diff --git a/.github/workflows/ci-matrix.yml b/.github/workflows/ci-matrix.yml index 5285afcbd..b444643e8 100644 --- a/.github/workflows/ci-matrix.yml +++ b/.github/workflows/ci-matrix.yml @@ -14,7 +14,7 @@ jobs: matrix: target: - { name: linux, os: ubuntu-22.04 } - - { name: macos, os: macos-13 } + - { name: macos, os: macos-latest } - { name: windows, os: windows-2022 } name: Build node on ${{ matrix.target.os }} diff --git a/.gitignore b/.gitignore index 88a2ab5f9..5416fd7bc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ **/*.rs.bk .idea/azure/ .idea/inspectionProfiles/Project_Default.xml +.idea/copilot* ### Node node_modules @@ -57,8 +58,10 @@ Temporary Items **/.idea/jsLibraryMappings.xml **/.idea/markdown-navigator.xml **/.idea/markdown-navigator/** +**/idea/db-forest-config.xml **/.idea/aws.xml **/.idea/scopes/*.xml +**/.idea/copilot* # Sensitive or high-churn files: **/.idea/**/dataSources/ diff --git a/.idea/SweepConfig.xml b/.idea/SweepConfig.xml new file mode 100644 index 000000000..bfb1652a3 --- /dev/null +++ b/.idea/SweepConfig.xml @@ -0,0 +1,67 @@ + + + + + + + \ No newline at end of file diff --git a/SWEEP.md b/SWEEP.md new file mode 100644 index 000000000..485f2958b --- /dev/null +++ b/SWEEP.md @@ -0,0 +1,549 @@ +# SWEEP.md - MASQ Node Development Guide + +This file contains essential information for AI assistants and developers working on the MASQ Node project, including coding standards, common commands, project structure, and development workflows. + +## Table of Contents +- [Project Overview](#project-overview) +- [Common Terminal Commands](#common-terminal-commands) +- [Project Structure](#project-structure) +- [Development Workflow](#development-workflow) +- [Code Style and Standards](#code-style-and-standards) +- [Testing Guidelines](#testing-guidelines) +- [Build and CI/CD](#build-and-cicd) +- [Important Notes](#important-notes) + +--- + +## Project Overview + +**MASQ Node** is the foundation of the MASQ Network, an open-source decentralized mesh-network (dMN) written primarily in Rust. It combines the benefits of VPN and Tor technology to create next-generation privacy software. + +### Key Technologies +- **Language**: Rust (2021 edition) +- **Build System**: Cargo +- **Testing**: Unit tests, integration tests, multinode integration tests (Docker-based) +- **CI/CD**: GitHub Actions +- **Platforms**: Linux, macOS, Windows (64-bit) + +### Repository Structure +The project is organized as a Cargo workspace with multiple crates: +- `node/` - Main MASQ Node implementation +- `masq/` - Command-line user interface +- `masq_lib/` - Shared library code +- `dns_utility/` - DNS configuration utility - deprecated +- `automap/` - Automatic public-IP detection and configuration +- `port_exposer/` - Port exposure utilities for testing only +- `ip_country/` - IP geolocation functionality +- `multinode_integration_tests/` - Docker-based integration tests employing multiple simultaneous Nodes + +--- + +## Common Terminal Commands + +### Building and Testing + +#### Run All Tests and Checks (Full CI Pipeline) +```bash +ci/all.sh +``` +This runs formatting, linting, unit tests, and integration tests for all components. **You will be prompted for your password** during zero-hop integration tests (requires sudo). + +#### Run Multinode Integration Tests (Linux only) +```bash +ci/multinode_integration_test.sh +``` +Note: These tests only run on Linux and require Docker. + +#### Format Code +```bash +ci/format.sh +``` +Formats all Rust code using `rustfmt`. This is required before submitting PRs. + +#### Run Linting (Clippy) +```bash +cargo clippy -- -D warnings -Anon-snake-case +``` +Or for a specific component: +```bash +cd node && ci/lint.sh +``` + +#### Run Unit Tests +```bash +# For the node component +cd node && ci/unit_tests.sh + +# Or manually +cargo test --release --lib --no-fail-fast --features masq_lib/log_recipient_test -- --nocapture --skip _integration +``` + +#### Run Integration Tests +```bash +cd node && ci/integration_tests.sh +``` +Note: Integration tests require sudo privileges on Linux and macOS. + +#### Build Release Version +```bash +cargo build --release +``` + +#### Build Debug Version +```bash +cargo build +``` + +### Version Management + +#### Bump Version +```bash +cd ci && ./bump_version.sh +``` +Example: +```bash +cd ci && ./bump_version.sh 6.9.1 +``` +This updates version numbers in all `Cargo.toml` files and corresponding `Cargo.lock` files. + +### Running MASQ Node + +#### Start MASQ Daemon (Linux/macOS) +```bash +sudo nohup ./MASQNode --initialization & +``` + +#### Start MASQ Daemon (Windows) +```bash +start /b MASQNode --initialization +``` + +#### Start MASQ CLI +```bash +./masq +``` + +#### Run MASQ Command (Non-interactive) +```bash +./masq setup --log-level debug --clandestine-port 1234 +``` + +#### Shutdown MASQ Node +```bash +./masq shutdown +``` + +#### Revert DNS Configuration +```bash +sudo ./dns_utility revert +``` + +### Git Workflow + +#### Update Master Branch +```bash +git checkout master +git pull +``` + +#### Create Feature Branch +```bash +git checkout -b GH- +git push -u origin HEAD +``` + +#### Merge Master into Feature Branch +```bash +git checkout master +git pull +git checkout - # Returns to previous branch +git merge master +``` + +### Docker Commands (for Multinode Tests) + +#### View Docker Logs +```bash +multinode_integration_tests/docker_logs.sh +``` + +#### Dump Docker State +```bash +multinode_integration_tests/docker_dump.sh +``` + +#### Build Docker Images +```bash +multinode_integration_tests/docker/build.sh +``` + +--- + +## Project Structure + +### Main Components + +#### Node (`node/`) +The core MASQ Node implementation containing: +- `src/accountant/` - Payment and accounting logic +- `src/blockchain/` - Blockchain interaction +- `src/entry_dns/` - Tiny DNS server that returns 'localhost' for any hostname - deprecated +- `src/hopper/` - Message routing +- `src/neighborhood/` - Network topology management +- `src/proxy_client/` - Exit-Node proxy +- `src/proxy_server/` - Originating-Node proxy +- `src/ui_gateway/` - User interface communication +- `src/sub_lib/` - Code shared among accountant, blockchain, hopper, etc. +- `src/test_utils/` - Testing utilities + +Each component has its own README.md with detailed documentation. + +#### MASQ CLI (`masq/`) +Command-line interface for controlling the MASQ Daemon and Node. + +#### DNS Utility (`dns_utility/`) - deprecated +The standard way to add intercept processing to your network data flow is to configure your system network stack to +use an HTTP and/or HTTPS proxy. However, early in the history of Substratum, Justin Tabb made a promise that Node +would be "zero-configuration." This utility, and the code in Node that it supports, were intended to keep that promise. +Essentially, Node contains a tiny DNS server that always returns 'localhost' for any hostname; therefore, when your +browser or other application performs DNS resolution, it will be fooled into routing its traffic through Node, which +is running on `localhost`. This utility wrangles your system DNS settings to point at that tiny DNS server, if you're +going to run Node, or back at your real DNS server if you're not. Since Node is now part of MASQ and no longer part of +Substratum and Justin Tabb is a part of history, the Node now operates as an HTTPS proxy and its `entry_dns` server is +no longer used. However, the code is still available and active and usable, so this utility remains available as well, +although its use is deprecated. + +#### Multinode Integration Tests (`multinode_integration_tests/`) +Docker-based integration tests that simulate multi-node networks. + +### Configuration Files + +#### Cargo.toml +Each component has its own `Cargo.toml` defining dependencies and metadata. + +#### config.toml +Runtime configuration file (located in data directory by default). +- Can be specified via `--config-file` parameter or `MASQ_CONFIG_FILE` environment variable +- Uses TOML format with scalar settings +- Example: `clandestine-port = 1234` + +--- + +## Development Workflow + +### Setting Up Development Environment + +1. **Install Rust toolchain**: + ```bash + rustup component add rustfmt + rustup component add clippy + ``` + +2. **Install Docker** (for multinode tests on Linux) + +3. **Clone repository and test**: + ```bash + git clone + cd Node + ci/all.sh + ``` + +### Working on an Issue + +1. **Select issue** from the [MASQ Node Card Wall](https://github.com/orgs/MASQ-Project/projects/1) +2. **Update master branch**: `git checkout master && git pull` +3. **Create feature branch**: `git checkout -b GH-` +4. **Complete the work** (test-driven development required) +5. **Merge in master regularly**: `git merge master` +6. **Run full test suite**: `ci/all.sh` +7. **Run multinode tests** (Linux only): `ci/multinode_integration_test.sh` +8. **Push changes**: `git push` +9. **Open pull request** on GitHub +10. **Watch GitHub Actions build** +11. **Address reviewer comments** +12. **Wait for QA approval** + +### Commit Guidelines + +- Commit frequently (commits will be squashed before merging) +- Commit when tests go green +- Commit before trying risky changes +- Write descriptive commit messages for your own reference + +--- + +## Code Style and Standards + +### Rust Standards + +#### Formatting +- Use `rustfmt` for all code formatting +- Run `ci/format.sh` before committing (non-auto-formatted code will fail CI) +- To skip formatting on specific code: `#[cfg_attr(rustfmt, rustfmt_skip)]` + +#### Linting +- All code must pass `clippy` with `-D warnings` +- Non-snake-case warnings are allowed: `-Anon-snake-case` +- Run `cargo clippy -- -D warnings -Anon-snake-case` + +#### Compiler Flags +```bash +export RUSTFLAGS="-D warnings -Anon-snake-case" +``` + +#### Error Handling +- Use `Result` types for fallible operations +- Provide descriptive error messages +- Handle errors appropriately (no `unwrap()` in production code; use `expect()` only, and that sparingly) +- **Important:** Nothing that comes into the system from the network must ever be allowed to cause a panic. All the +standard anti-injection rules apply, but whenever something from the outside generates an error, the error must be +logged and execution must continue. (Sometimes additional action is appropriate, such as banning an evil Node; but +logging the error is the bare minimum.) `eprintln!()` is not logging. + +#### Testing +- **Test-Driven Development (TDD) is required** +- All new code must have corresponding tests +- Tests must pass before code review +- Use descriptive test names +- Test both success and failure cases + +### Naming Conventions +- Use snake_case for functions and variables +- Use PascalCase for types and traits +- Use SCREAMING_SNAKE_CASE for constants +- All zero-hop integration tests must have names _suffixed_ with `_integration`; otherwise they'll run as unit tests +rather than integration tests. + +### Documentation +- Document public APIs with doc comments (`///`) +- Include examples in doc comments where helpful +- Keep README.md files updated for each component + +--- + +## Testing Guidelines + +### Test Types + +#### Unit Tests +- Located in the same file as the code being tested (in `#[cfg(test)]` modules) +- Test individual functions and methods in isolation +- Run with: `cargo test --lib` +- Should not require sudo or network access + +#### Integration Tests +- Test interactions between components +- Starts up a real Node, always in `--neighborhood-mode zero-hop` +- Will require sudo privileges, because a Node has to start with root privileges to open low ports +- Run with: `ci/integration_tests.sh` + +#### Multinode Integration Tests +- Docker-based tests simulating multi-node networks +- **Linux only** (do not run on macOS or Windows) +- Require Docker installation +- Run with: `ci/multinode_integration_test.sh` + +### Test Execution + +#### Run All Tests +```bash +ci/all.sh +``` + +#### Run Specific Test +```bash +cargo test test_name -- --nocapture +``` + +#### Run Tests with Backtrace +```bash +RUST_BACKTRACE=full cargo test +``` + +#### Run Tests in Release Mode +```bash +cargo test --release +``` + +### Test Features +- Use `--no-fail-fast` to run all tests even if some fail +- Use `--nocapture` to see println! output + +--- + +## Build and CI/CD + +### Local Build Process + +The `ci/all.sh` script performs the following: +1. Format check and auto-format (`ci/format.sh`) +2. Install git hooks (`install-hooks.sh`) +3. Start sccache server (for faster compilation) +4. Build and test each component: + - masq_lib + - node + - dns_utility + - masq (CLI) + - automap + - ip_country + +### GitHub Actions + +- Builds run automatically on pull requests +- Unit and single-Node integration tests run on Linux, macOS, and Windows +- Multinode integration tests run on Linux only +- All builds must pass before merging + +### Build Artifacts + +After successful builds, artifacts are available in: +- `/target/release/` and `/target/debug/` - Executable binaries + - `MASQNode` (or `MASQNode.exe`) - Node and Daemon + - `masq` - Command-line interface + - `dns_utility` - DNS configuration tool + - `automap` - Firewall penetration test utility: requires special remote setup + +### Caching + +The project uses `sccache` for faster compilation: +```bash +export SCCACHE_DIR="$HOME/.cargo/sccache" +SCCACHE_IDLE_TIMEOUT=0 sccache --start-server +``` + +--- + +## Important Notes + +### Platform-Specific Considerations + +#### Linux +- Full support for all features +- Multinode integration tests available +- Requires sudo for integration tests +- May need to free port 53: `sudo ci/free-port-53.sh` + +#### macOS +- Full support except multinode integration tests +- Requires sudo for integration tests +- May need to adjust file descriptor limits in GitHub Actions + +#### Windows +- Use Git Bash for running scripts +- Multinode integration tests not supported +- Some services may need to be stopped (ICS, W3SVC) +- 32-bit Windows not reliably supported (use 64-bit) +- Run as Administrator + +### Security Considerations + +- **MASQ Node is currently in beta** - not clandestine yet +- Do not use for sensitive traffic +- Traffic cannot be decrypted by attackers but MASQ traffic is identifiable +- Database encryption uses a user-provided password +- Password is never stored on disk +- Forgetting the password means losing all the encrypted content in the database and starting over + +### Configuration Priority + +Configuration sources in order of priority (highest to lowest): +1. `masq` UI commands +2. Environment variables (prefixed with `MASQ_`) +3. Configuration file (`config.toml`) +4. Defaults + +Example: +- CLI, non-interactive mode: `masq setup --clandestine-port 1234` +- Environment: `MASQ_CLANDESTINE_PORT=1234` +- Config file: `clandestine-port = "1234"` + +### Daemon vs Node + +- **Daemon**: Runs with admin privileges, starts at boot, cannot access network or communicate with Node +- **Node**: Starts with admin privileges, drops privileges after network configuration, handles all network traffic +- UI connects to Daemon first, then is redirected to Node when it first sends the Daemon a Node command + +### Error Handling in Browser + +#### HTTP Errors +- MASQ Node can impersonate the remote server for HTTP errors +- Errors are clearly marked as coming from MASQ Node rather than the remote server + +#### TLS Errors +- In-band error reporting severely limited by TLS protocol; only available for ClientHello +- Friendliest errors are written to the log + +### Development Tools + +#### Recommended IDE +- JetBrains IntelliJ IDEA with Rust plugin (used by MASQ team) +- Other options: VS Code with rust-analyzer, etc. + +#### Required Tools +- Rust toolchain (rustc, cargo, rustfmt, clippy) +- Git +- Docker (for multinode tests on Linux) +- sudo access (for integration tests) + +### Useful Links + +- [MASQ Node Repository](https://github.com/MASQ-Project/Node) +- [MASQ Node Card Wall](https://github.com/orgs/MASQ-Project/projects/1) +- [GitHub Actions Build Site](https://github.com/MASQ-Project/Node/actions) +- [Knowledge Base](https://docs.masq.ai/masq) +- [Discord Channel](https://discord.gg/masq) +- [Latest Release](https://github.com/MASQ-Project/Node/releases/latest) + +--- + +## Quick Reference + +### Most Common Commands + +```bash +# Full test suite +ci/all.sh + +# Format code +ci/format.sh + +# Lint code +cargo clippy -- -D warnings -Anon-snake-case + +# Run unit tests +cargo test --release --lib --no-fail-fast + +# Build release +cargo build --release + +# Start daemon (Linux/macOS) +sudo nohup ./MASQNode --initialization & + +# Start CLI +./masq + +# Shutdown node +./masq shutdown +``` + +### Environment Variables + +```bash +export RUST_BACKTRACE=full # Full backtraces on panic +export RUSTFLAGS="-D warnings -Anon-snake-case" # Compiler flags +export SCCACHE_DIR="$HOME/.cargo/sccache" # Cache directory +``` + +### File Locations + +- Binaries: `/target/release/` or `/target/debug/` +- Config file: `/config.toml` +- Database: `/` +- Logs: Configured via `--log-level` parameter + +--- + +**Last Updated**: 2026 +**Project**: MASQ Node +**License**: GPL-3.0-only +**Copyright**: (c) 2026, MASQ Network diff --git a/automap/Cargo.lock b/automap/Cargo.lock index 82258e87b..409a66cff 100644 --- a/automap/Cargo.lock +++ b/automap/Cargo.lock @@ -137,7 +137,7 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "automap" -version = "0.9.0" +version = "0.9.1" dependencies = [ "crossbeam-channel 0.5.8", "flexi_logger", @@ -942,7 +942,7 @@ dependencies = [ [[package]] name = "ip_country" -version = "0.1.0" +version = "0.9.1" dependencies = [ "csv", "ipnetwork", @@ -1116,7 +1116,7 @@ dependencies = [ [[package]] name = "masq_lib" -version = "0.9.0" +version = "0.9.1" dependencies = [ "actix", "clap", @@ -2043,7 +2043,7 @@ dependencies = [ [[package]] name = "test_utilities" -version = "0.1.0" +version = "0.9.1" [[package]] name = "textwrap" diff --git a/automap/Cargo.toml b/automap/Cargo.toml index b379c8a55..f4fa870de 100644 --- a/automap/Cargo.toml +++ b/automap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "automap" -version = "0.9.0" +version = "0.9.1" authors = ["Dan Wiebe ", "MASQ"] license = "GPL-3.0-only" description = "Library full of code to make routers map ports through firewalls" diff --git a/dns_utility/Cargo.lock b/dns_utility/Cargo.lock index 32431ab07..7f87dbdb9 100644 --- a/dns_utility/Cargo.lock +++ b/dns_utility/Cargo.lock @@ -457,7 +457,7 @@ dependencies = [ [[package]] name = "dns_utility" -version = "0.9.0" +version = "0.9.1" dependencies = [ "core-foundation", "ipconfig 0.2.2", @@ -757,7 +757,7 @@ dependencies = [ [[package]] name = "ip_country" -version = "0.1.0" +version = "0.9.1" dependencies = [ "csv", "ipnetwork", @@ -919,7 +919,7 @@ dependencies = [ [[package]] name = "masq_lib" -version = "0.9.0" +version = "0.9.1" dependencies = [ "actix", "clap", @@ -1775,7 +1775,7 @@ dependencies = [ [[package]] name = "test_utilities" -version = "0.1.0" +version = "0.9.1" [[package]] name = "textwrap" diff --git a/dns_utility/Cargo.toml b/dns_utility/Cargo.toml index c21f8a9ca..beabe18f5 100644 --- a/dns_utility/Cargo.toml +++ b/dns_utility/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dns_utility" -version = "0.9.0" +version = "0.9.1" license = "GPL-3.0-only" authors = ["Dan Wiebe ", "MASQ"] copyright = "Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved." diff --git a/ip_country/Cargo.toml b/ip_country/Cargo.toml index e974fcf41..ce74c5f48 100644 --- a/ip_country/Cargo.toml +++ b/ip_country/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ip_country" -version = "0.1.0" +version = "0.9.1" edition = "2021" license = "GPL-3.0-only" authors = ["Dan Wiebe ", "MASQ"] diff --git a/masq/Cargo.toml b/masq/Cargo.toml index 0bef7d9ca..878b4e172 100644 --- a/masq/Cargo.toml +++ b/masq/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "masq" -version = "0.9.0" +version = "0.9.1" authors = ["Dan Wiebe ", "MASQ"] license = "GPL-3.0-only" description = "Reference implementation of user interface for MASQ Node" diff --git a/masq/src/commands/configuration_command.rs b/masq/src/commands/configuration_command.rs index 62d2b4650..2442270a6 100644 --- a/masq/src/commands/configuration_command.rs +++ b/masq/src/commands/configuration_command.rs @@ -166,8 +166,8 @@ impl ConfigurationCommand { let scan_intervals = Self::preprocess_combined_parameters({ let s_i = &configuration.scan_intervals; &[ - ("Pending payable:", &s_i.pending_payable_sec, "s"), ("Payable:", &s_i.payable_sec, "s"), + ("Pending payable:", &s_i.pending_payable_sec, "s"), ("Receivable:", &s_i.receivable_sec, "s"), ] }); @@ -397,8 +397,8 @@ mod tests { | Exit byte rate: 129,000,000 wei\n\ | Exit service rate: 160,000,000 wei\n\ |Scan intervals: \n\ -| Pending payable: 150,500 s\n\ | Payable: 155,000 s\n\ +| Pending payable: 150,500 s\n\ | Receivable: 250,666 s\n" ) .replace('|', "") @@ -494,8 +494,8 @@ mod tests { | Exit byte rate: 20 wei\n\ | Exit service rate: 30 wei\n\ |Scan intervals: \n\ -| Pending payable: 1,000 s\n\ | Payable: 1,000 s\n\ +| Pending payable: 1,000 s\n\ | Receivable: 1,000 s\n", ) .replace('|', "") diff --git a/masq_lib/Cargo.toml b/masq_lib/Cargo.toml index ca2ce815b..94ceadb17 100644 --- a/masq_lib/Cargo.toml +++ b/masq_lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "masq_lib" -version = "0.9.0" +version = "0.9.1" authors = ["Dan Wiebe ", "MASQ"] license = "GPL-3.0-only" description = "Code common to Node and masq; also, temporarily, to dns_utility" diff --git a/masq_lib/src/constants.rs b/masq_lib/src/constants.rs index 20e332f4a..1cc14f357 100644 --- a/masq_lib/src/constants.rs +++ b/masq_lib/src/constants.rs @@ -5,7 +5,7 @@ use crate::data_version::DataVersion; use const_format::concatcp; pub const DEFAULT_CHAIN: Chain = Chain::BaseMainnet; -pub const CURRENT_SCHEMA_VERSION: usize = 12; +pub const CURRENT_SCHEMA_VERSION: usize = 13; pub const HIGHEST_RANDOM_CLANDESTINE_PORT: u16 = 9999; pub const HTTP_PORT: u16 = 80; @@ -83,7 +83,8 @@ pub const VALUE_EXCEEDS_ALLOWED_LIMIT: u64 = ACCOUNTANT_PREFIX | 3; pub const MASQ_TOTAL_SUPPLY: u64 = 37_500_000; pub const DEFAULT_GAS_PRICE: u64 = 1; //TODO ?? Really -pub const DEFAULT_GAS_PRICE_MARGIN: u64 = 30; +pub const DEFAULT_GAS_PRICE_RETRY_PERCENTAGE: u64 = 30; +pub const DEFAULT_GAS_PRICE_RETRY_CONSTANT: u128 = 5_000; pub const DEFAULT_MAX_BLOCK_COUNT: u64 = 100_000; //chains @@ -142,7 +143,8 @@ mod tests { assert_eq!(CURRENT_LOGFILE_NAME, "MASQNode_rCURRENT.log"); assert_eq!(MASQ_PROMPT, "masq> "); assert_eq!(DEFAULT_GAS_PRICE, 1); - assert_eq!(DEFAULT_GAS_PRICE_MARGIN, 30); + assert_eq!(DEFAULT_GAS_PRICE_RETRY_PERCENTAGE, 30); + assert_eq!(DEFAULT_GAS_PRICE_RETRY_CONSTANT, 5_000); assert_eq!(WALLET_ADDRESS_LENGTH, 42); assert_eq!(MASQ_TOTAL_SUPPLY, 37_500_000); assert_eq!(WEIS_IN_GWEI, 1_000_000_000); diff --git a/masq_lib/src/logger.rs b/masq_lib/src/logger.rs index b5fe282c8..5a94127b0 100644 --- a/masq_lib/src/logger.rs +++ b/masq_lib/src/logger.rs @@ -116,6 +116,10 @@ impl Logger { } } + pub fn name(&self) -> &str { + &self.name + } + pub fn trace(&self, log_function: F) where F: FnOnce() -> String, diff --git a/masq_lib/src/shared_schema.rs b/masq_lib/src/shared_schema.rs index a273205ff..b99a79717 100644 --- a/masq_lib/src/shared_schema.rs +++ b/masq_lib/src/shared_schema.rs @@ -7,7 +7,7 @@ use crate::constants::{ POLYGON_MAINNET_FULL_IDENTIFIER, }; use crate::crash_point::CrashPoint; -use clap::{App, Arg}; +use clap::{arg_enum, App, Arg}; use lazy_static::lazy_static; pub const BLOCKCHAIN_SERVICE_HELP: &str = @@ -81,6 +81,17 @@ pub const NEIGHBORS_HELP: &str = "One or more Node descriptors for running Nodes if you don't specify a neighbor, your Node will start without being connected to any MASQ \ Network, although other Nodes will be able to connect to yours if they know your Node's descriptor. \ --neighbors is meaningless in --neighborhood-mode zero-hop."; +pub const NEW_PUBLIC_KEY_HELP: &str = "Whenever you start it, the Node will try to use the same public key \ + it used last time. That's '--new-public-key off'. If you want it to select a new public key when it \ + starts, then specify '--new-public-key on', and you'll get a different one this time...which it will \ + reuse next time unless you specify '--new-public-key on' again.\n\n\ + You should be careful about restarting your Node with the same public key too quickly. If your new \ + Node tries to join the Network before the Network has forgotten your old Node, every Node you try \ + to connect to will ignore you.\n\n\ + There are some conditions under which the Node cannot use the same public key it used last time: \ + for example, if there was no last time, or if you don't specify a `--db-password`. Normally, in \ + these situations, the Node will select a new public key and store it for future use; but if you \ + explicitly demand the old public key with `--new-public-key off`, the Node will refuse to start."; // generated valid encoded keys for future needs // UJNoZW5p/PDVqEjpr3b+8jZ/93yPG8i5dOAgE1bhK+A @@ -225,6 +236,14 @@ lazy_static! { DEFAULT_GAS_PRICE); } +arg_enum! { + #[derive(Debug, PartialEq, Eq, Clone, Copy)] + pub enum OnOff { + On, + Off, + } +} + // These Args are needed in more than one clap schema. To avoid code duplication, they're defined here and referred // to from multiple places. pub fn chain_arg<'a>() -> Arg<'a, 'a> { @@ -466,13 +485,23 @@ pub fn shared_app(head: App<'static, 'static>) -> App<'static, 'static> { .min_values(0) .help(NEIGHBORS_HELP), ) + .arg( + Arg::with_name("new-public-key") + .long("new-public-key") + .value_name("NEW-PUBLIC-KEY") + .takes_value(true) + .possible_values(&OnOff::variants()) + .case_insensitive(true) + .help(NEW_PUBLIC_KEY_HELP), + ) .arg(real_user_arg()) .arg( Arg::with_name("scans") .long("scans") .value_name("SCANS") .takes_value(true) - .possible_values(&["on", "off"]) + .possible_values(&OnOff::variants()) + .case_insensitive(true) .help(SCANS_HELP), ) .arg(common_parameter_with_separate_u64_values( diff --git a/multinode_integration_tests/Cargo.toml b/multinode_integration_tests/Cargo.toml index 6a9d22533..d3070e920 100644 --- a/multinode_integration_tests/Cargo.toml +++ b/multinode_integration_tests/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "multinode_integration_tests" -version = "0.9.0" +version = "0.9.1" authors = ["Dan Wiebe ", "MASQ"] license = "GPL-3.0-only" description = "" diff --git a/multinode_integration_tests/src/masq_node_cluster.rs b/multinode_integration_tests/src/masq_node_cluster.rs index 86a94af54..e233f6674 100644 --- a/multinode_integration_tests/src/masq_node_cluster.rs +++ b/multinode_integration_tests/src/masq_node_cluster.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::env; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4, ToSocketAddrs}; +use std::time::Duration; pub struct MASQNodeCluster { startup_configs: HashMap<(String, usize), NodeStartupConfig>, @@ -306,6 +307,42 @@ impl MASQNodeCluster { } fn create_network() -> Result<(), String> { + let mut errors = vec![]; + let mut retries_remaining = 2; + let interval = Duration::from_millis(250); + while retries_remaining >= 0 { + match MASQNodeCluster::create_network_attempt() { + Ok(s) => return Ok(s), + Err(err_msg) => { + retries_remaining -= 1; + errors.push(err_msg); + std::thread::sleep(interval); + } + } + } + Err(format!( + "Errors trying to create Docker network:\n {}\n", + errors.join("\n ") + )) + } + + fn create_network_attempt() -> Result<(), String> { + let mut command = Command::new( + "docker", + Command::strings(vec!["network", "rm", "integration_net"]), + ); + match command.stdout_or_stderr() { + Ok(_) => println!("Removed existing integration_net network"), + Err(msg) if msg.contains("network integration_net not found") => { + println!("No existing integration_net network to remove: cool!") + } + Err(msg) => { + return Err(format!( + "Error removing existing integration_net network: {}", + msg + )) + } + } let mut command = Command::new( "docker", Command::strings(vec![ diff --git a/multinode_integration_tests/tests/bookkeeping_test.rs b/multinode_integration_tests/tests/bookkeeping_test.rs index de716a3e3..c2845fc72 100644 --- a/multinode_integration_tests/tests/bookkeeping_test.rs +++ b/multinode_integration_tests/tests/bookkeeping_test.rs @@ -1,4 +1,5 @@ // Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. +use itertools::Itertools; use multinode_integration_tests_lib::masq_node::{MASQNode, NodeReference}; use multinode_integration_tests_lib::masq_node_cluster::MASQNodeCluster; use multinode_integration_tests_lib::masq_real_node::{ @@ -30,54 +31,72 @@ fn provided_and_consumed_services_are_recorded_in_databases() { let mut client = originating_node.make_client(8080, STANDARD_CLIENT_TIMEOUT_MILLIS); client.set_timeout(Duration::from_secs(10)); - let request = "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n".as_bytes(); + let request = "GET /index.html HTTP/1.1\r\nHost: www.testingmcafeesites.com\r\n\r\n".as_bytes(); client.send_chunk(request); let response = String::from_utf8(client.wait_for_chunk()).unwrap(); assert!( - response.contains("

Example Domain

"), - "Not from www.example.com:\n{}", + response.contains("URL for testing."), + "Not from www.testingmcafeesites.com:\n{}", response ); + // Waiting until everybody has finished generating payables and receivables + thread::sleep(Duration::from_secs(10)); + // get all payables from originating node let payables = retrieve_payables(&originating_node); - // Waiting until the serving nodes have finished accruing their receivables - thread::sleep(Duration::from_secs(10)); - // get all receivables from all other nodes - let receivable_balances = non_originating_nodes + let receivable_nodes = non_originating_nodes .iter() .flat_map(|node| { receivables(node) .into_iter() .map(move |receivable_account| { - (node.earning_wallet(), receivable_account.balance_wei) + ( + node.earning_wallet(), + (node.name().to_string(), receivable_account.balance_wei), + ) }) }) - .collect::>(); + .collect::>(); // check that each payable has a receivable assert_eq!( payables.len(), - receivable_balances.len(), + receivable_nodes.len(), "Lengths of payables and receivables should match.\nPayables: {:?}\nReceivables: {:?}", payables, - receivable_balances + receivable_nodes ); assert!( - receivable_balances.len() >= 3, // minimum service list: route, route, exit. + receivable_nodes.len() >= 3, // minimum service list: route, route, exit. "not enough receivables found {:?}", - receivable_balances + receivable_nodes ); - payables.iter().for_each(|payable| { - assert_eq!( - payable.balance_wei, - *receivable_balances.get(&payable.wallet).unwrap() as u128, - ); - }); + let messages = payables + .iter() + .flat_map(|payable| { + let payable_balance = payable.balance_wei; + let (non_originating_node_name, receivable_balance) = + receivable_nodes.get(&payable.wallet).unwrap().clone(); + if payable_balance != receivable_balance as u128 { + Some(format!( + "Payable for {} ({}) does not match receivable for {} ({})", + originating_node.name(), + payable_balance, + non_originating_node_name, + receivable_balance + )) + } else { + None + } + }) + .collect_vec(); + + assert!(messages.is_empty(), "{:#?}", messages); } fn retrieve_payables(node: &MASQRealNode) -> Vec { diff --git a/multinode_integration_tests/tests/connection_progress_test.rs b/multinode_integration_tests/tests/connection_progress_test.rs index 336cef2c2..fa307fdf5 100644 --- a/multinode_integration_tests/tests/connection_progress_test.rs +++ b/multinode_integration_tests/tests/connection_progress_test.rs @@ -57,7 +57,7 @@ fn connection_progress_is_properly_broadcast() { .collect::>(); let message_body = - ui_client.wait_for_specific_broadcast(vec!["connectionChange"], Duration::from_secs(5)); + ui_client.wait_for_specific_broadcast(vec!["connectionChange"], Duration::from_secs(10)); let (ccb, _) = UiConnectionChangeBroadcast::fmb(message_body).unwrap(); if ccb.stage == UiConnectionStage::ConnectedToNeighbor { let message_body = diff --git a/multinode_integration_tests/tests/connection_termination_test.rs b/multinode_integration_tests/tests/connection_termination_test.rs index 2657661b8..287f2f120 100644 --- a/multinode_integration_tests/tests/connection_termination_test.rs +++ b/multinode_integration_tests/tests/connection_termination_test.rs @@ -1,6 +1,7 @@ // Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. use masq_lib::blockchains::chains::Chain; +use masq_lib::constants::HTTP_PORT; use masq_lib::test_utils::utils::TEST_DEFAULT_MULTINODE_CHAIN; use masq_lib::utils::find_free_port; use multinode_integration_tests_lib::masq_mock_node::MASQMockNode; @@ -86,13 +87,12 @@ fn reported_server_drop() { let (_, _, lcp) = mock_node .wait_for_package(&masquerader, Duration::from_secs(2)) .unwrap(); - let (stream_key, return_route_id) = - context_from_request_lcp(lcp, real_node.main_cryptde_null().unwrap(), &exit_cryptde); + let stream_key = stream_key_from_request_lcp(lcp, &exit_cryptde); mock_node .transmit_package( mock_node.port_list()[0], - create_server_drop_report(&mock_node, &real_node, stream_key, return_route_id), + create_server_drop_report(&mock_node, &real_node, stream_key), &masquerader, real_node.main_public_key(), real_node.socket_addr(PortSelector::First), @@ -115,7 +115,7 @@ fn actual_server_drop() { let server_port = find_free_port(); let mut server = real_node.make_server(server_port); let masquerader = JsonMasquerader::new(); - let (stream_key, return_route_id) = arbitrary_context(); + let stream_key = arbitrary_stream_key(); let index: u64 = 0; request_server_payload( index, @@ -125,7 +125,6 @@ fn actual_server_drop() { &mut server, &masquerader, stream_key, - return_route_id, ); let index: u64 = 1; request_server_payload( @@ -136,7 +135,6 @@ fn actual_server_drop() { &mut server, &masquerader, stream_key, - return_route_id, ); server.shutdown(); @@ -174,7 +172,6 @@ fn request_server_payload( server: &mut MASQNodeServer, masquerader: &JsonMasquerader, stream_key: StreamKey, - return_route_id: u32, ) { mock_node .transmit_package( @@ -184,7 +181,6 @@ fn request_server_payload( &mock_node, &real_node, stream_key, - return_route_id, &server, cluster.chain, ), @@ -212,7 +208,7 @@ fn reported_client_drop() { let server_port = find_free_port(); let mut server = real_node.make_server(server_port); let masquerader = JsonMasquerader::new(); - let (stream_key, return_route_id) = arbitrary_context(); + let stream_key = arbitrary_stream_key(); let index: u64 = 0; mock_node .transmit_package( @@ -222,7 +218,6 @@ fn reported_client_drop() { &mock_node, &real_node, stream_key, - return_route_id, &server, cluster.chain, ), @@ -240,7 +235,7 @@ fn reported_client_drop() { mock_node .transmit_package( mock_node.port_list()[0], - create_client_drop_report(&mock_node, &real_node, stream_key, return_route_id), + create_client_drop_report(&mock_node, &real_node, stream_key), &masquerader, real_node.main_public_key(), real_node.socket_addr(PortSelector::First), @@ -322,11 +317,7 @@ fn full_neighbor(one: &mut NodeRecord, another: &mut NodeRecord) { .unwrap(); } -fn context_from_request_lcp( - lcp: LiveCoresPackage, - originating_cryptde: &dyn CryptDE, - exit_cryptde: &dyn CryptDE, -) -> (StreamKey, u32) { +fn stream_key_from_request_lcp(lcp: LiveCoresPackage, exit_cryptde: &dyn CryptDE) -> StreamKey { let payload = match decodex::(exit_cryptde, &lcp.payload).unwrap() { MessageType::ClientRequest(vd) => vd .extract(&node_lib::sub_lib::migrations::client_request_payload::MIGRATIONS) @@ -334,15 +325,11 @@ fn context_from_request_lcp( mt => panic!("Unexpected: {:?}", mt), }; let stream_key = payload.stream_key; - let return_route_id = decodex::(originating_cryptde, &lcp.route.hops[6]).unwrap(); - (stream_key, return_route_id) + stream_key } -fn arbitrary_context() -> (StreamKey, u32) { - ( - StreamKey::make_meaningful_stream_key("arbitrary_context"), - 12345678, - ) +fn arbitrary_stream_key() -> StreamKey { + StreamKey::make_meaningful_stream_key("arbitrary_context") } fn create_request_icp( @@ -350,12 +337,12 @@ fn create_request_icp( originating_node: &MASQMockNode, exit_node: &MASQRealNode, stream_key: StreamKey, - return_route_id: u32, server: &MASQNodeServer, chain: Chain, ) -> IncipientCoresPackage { + let originating_main_cryptde = originating_node.main_cryptde_null().unwrap(); IncipientCoresPackage::new( - originating_node.main_cryptde_null().unwrap(), + originating_main_cryptde, Route::round_trip( RouteSegment::new( vec![ @@ -371,9 +358,8 @@ fn create_request_icp( ], Component::ProxyServer, ), - originating_node.main_cryptde_null().unwrap(), + originating_main_cryptde, originating_node.consuming_wallet(), - return_route_id, Some(chain.rec().contract), ) .unwrap(), @@ -382,7 +368,7 @@ fn create_request_icp( &ClientRequestPayload_0v1 { stream_key, sequenced_packet: SequencedPacket::new(Vec::from(HTTP_REQUEST), index, false), - target_hostname: Some(format!("{}", server.local_addr().ip())), + target_hostname: format!("{}", server.local_addr().ip()), target_port: server.local_addr().port(), protocol: ProxyProtocol::HTTP, originator_public_key: originating_node.main_public_key().clone(), @@ -400,8 +386,9 @@ fn create_meaningless_icp( let socket_addr = SocketAddr::from_str("3.2.1.0:7654").unwrap(); let stream_key = StreamKey::make_meaningful_stream_key("Chancellor on brink of second bailout for banks"); + let main_cryptde = originating_node.main_cryptde_null().unwrap(); IncipientCoresPackage::new( - originating_node.main_cryptde_null().unwrap(), + main_cryptde, Route::round_trip( RouteSegment::new( vec![ @@ -417,9 +404,8 @@ fn create_meaningless_icp( ], Component::ProxyServer, ), - originating_node.main_cryptde_null().unwrap(), + main_cryptde, originating_node.consuming_wallet(), - 1357, Some(TEST_DEFAULT_MULTINODE_CHAIN.rec().contract), ) .unwrap(), @@ -428,7 +414,7 @@ fn create_meaningless_icp( &ClientRequestPayload_0v1 { stream_key, sequenced_packet: SequencedPacket::new(Vec::from(HTTP_REQUEST), 0, false), - target_hostname: Some(format!("nowhere.com")), + target_hostname: "nowhere.com".to_string(), target_port: socket_addr.port(), protocol: ProxyProtocol::HTTP, originator_public_key: originating_node.main_public_key().clone(), @@ -443,8 +429,9 @@ fn create_server_drop_report( exit_node: &MASQMockNode, originating_node: &MASQRealNode, stream_key: StreamKey, - return_route_id: u32, ) -> IncipientCoresPackage { + let exit_main_cryptde = exit_node.main_cryptde_null().unwrap(); + let originating_main_cryptde = originating_node.main_cryptde_null().unwrap(); let mut route = Route::round_trip( RouteSegment::new( vec![ @@ -460,15 +447,12 @@ fn create_server_drop_report( ], Component::ProxyServer, ), - originating_node.main_cryptde_null().unwrap(), + originating_main_cryptde, originating_node.consuming_wallet(), - return_route_id, Some(TEST_DEFAULT_MULTINODE_CHAIN.rec().contract), ) .unwrap(); - route - .shift(originating_node.main_cryptde_null().unwrap()) - .unwrap(); + route.shift(originating_main_cryptde).unwrap(); let payload = MessageType::ClientResponse(VersionedData::new( &node_lib::sub_lib::migrations::client_response_payload::MIGRATIONS, &ClientResponsePayload_0v1 { @@ -478,7 +462,7 @@ fn create_server_drop_report( )); IncipientCoresPackage::new( - exit_node.main_cryptde_null().unwrap(), + exit_main_cryptde, route, payload, originating_node.alias_public_key(), @@ -490,8 +474,8 @@ fn create_client_drop_report( originating_node: &MASQMockNode, exit_node: &MASQRealNode, stream_key: StreamKey, - return_route_id: u32, ) -> IncipientCoresPackage { + let originating_main_cryptde = originating_node.main_cryptde_null().unwrap(); let route = Route::round_trip( RouteSegment::new( vec![ @@ -507,9 +491,8 @@ fn create_client_drop_report( ], Component::ProxyServer, ), - originating_node.main_cryptde_null().unwrap(), + originating_main_cryptde, originating_node.consuming_wallet(), - return_route_id, Some(TEST_DEFAULT_MULTINODE_CHAIN.rec().contract), ) .unwrap(); @@ -518,15 +501,15 @@ fn create_client_drop_report( &ClientRequestPayload_0v1 { stream_key, sequenced_packet: SequencedPacket::new(vec![], 1, true), - target_hostname: Some(String::from("doesnt.matter.com")), - target_port: 80, + target_hostname: String::from("doesnt.matter.com"), + target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: originating_node.main_public_key().clone(), }, )); IncipientCoresPackage::new( - originating_node.main_cryptde_null().unwrap(), + originating_main_cryptde, route, payload, exit_node.main_public_key(), diff --git a/multinode_integration_tests/tests/data_routing_test.rs b/multinode_integration_tests/tests/data_routing_test.rs index 98dd460a1..5d3a0076e 100644 --- a/multinode_integration_tests/tests/data_routing_test.rs +++ b/multinode_integration_tests/tests/data_routing_test.rs @@ -58,11 +58,11 @@ fn http_end_to_end_routing_test() { thread::sleep(Duration::from_millis(500)); let mut client = last_node.make_client(8080, 5000); - client.send_chunk(b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"); + client.send_chunk(b"GET /index.html HTTP/1.1\r\nHost: www.testingmcafeesites.com\r\n\r\n"); let response = client.wait_for_chunk(); assert_eq!( - index_of(&response, &b"

Example Domain

"[..]).is_some(), + index_of(&response, &b"URL for testing."[..]).is_some(), true, "Actual response:\n{}", String::from_utf8(response).unwrap() @@ -89,7 +89,7 @@ fn http_end_to_end_routing_test_with_consume_and_originate_only_nodes() { .chain(cluster.chain) .build(), ); - let _potential_exit_nodes = vec![0, 1, 2, 3, 4] + let _potential_exit_nodes = vec![3, 4, 5, 6, 7] .into_iter() .map(|_| { cluster.start_real_node( @@ -101,17 +101,19 @@ fn http_end_to_end_routing_test_with_consume_and_originate_only_nodes() { }) .collect_vec(); - thread::sleep(Duration::from_millis(1000)); + thread::sleep(Duration::from_secs(5)); let mut client = originating_node.make_client(8080, STANDARD_CLIENT_TIMEOUT_MILLIS); - client.send_chunk(b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"); - let response = client.wait_for_chunk(); + client.set_timeout(Duration::from_secs(10)); + let request = "GET /index.html HTTP/1.1\r\nHost: www.testingmcafeesites.com\r\n\r\n".as_bytes(); - assert_eq!( - index_of(&response, &b"

Example Domain

"[..]).is_some(), - true, - "Actual response:\n{}", - String::from_utf8(response).unwrap() + client.send_chunk(request); + let response = String::from_utf8(client.wait_for_chunk()).unwrap(); + + assert!( + response.contains("URL for testing."), + "Not from www.testingmcafeesites.com:\n{}", + response ); } @@ -235,11 +237,11 @@ fn http_routing_failure_produces_internal_error_response() { let mut client = originating_node.make_client(8080, STANDARD_CLIENT_TIMEOUT_MILLIS); - client.send_chunk(b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"); + client.send_chunk(b"GET /index.html HTTP/1.1\r\nHost: www.testingmcafeesites.com\r\n\r\n"); let response = client.wait_for_chunk(); let expected_response = - ServerImpersonatorHttp {}.route_query_failure_response("www.example.com"); + ServerImpersonatorHttp {}.route_query_failure_response("www.testingmcafeesites.com"); assert_eq!( &expected_response, @@ -315,27 +317,56 @@ fn multiple_stream_zero_hop_test() { let mut one_client = zero_hop_node.make_client(8080, STANDARD_CLIENT_TIMEOUT_MILLIS); let mut another_client = zero_hop_node.make_client(8080, STANDARD_CLIENT_TIMEOUT_MILLIS); - one_client.send_chunk(b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"); - another_client.send_chunk(b"GET / HTTP/1.1\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\r\nAccept-Language: cs-CZ,cs;q=0.9,en;q=0.8,sk;q=0.7\r\nCache-Control: max-age=0\r\nConnection: keep-alive\r\nHost: www.testingmcafeesites.com\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36\r\n\r\n"); + one_client.send_chunk(b"GET /testcat_ed.html HTTP/1.1\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\r\nAccept-Language: cs-CZ,cs;q=0.9,en;q=0.8,sk;q=0.7\r\nCache-Control: max-age=0\r\nConnection: keep-alive\r\nHost: www.testingmcafeesites.com\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36\r\n\r\n"); + another_client.send_chunk(b"GET /testcat_cm.html HTTP/1.1\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7\r\nAccept-Language: cs-CZ,cs;q=0.9,en;q=0.8,sk;q=0.7\r\nCache-Control: max-age=0\r\nConnection: keep-alive\r\nHost: www.testingmcafeesites.com\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36\r\n\r\n"); let one_response = one_client.wait_for_chunk(); let another_response = another_client.wait_for_chunk(); - assert_eq!( - index_of(&one_response, &b"

Example Domain

"[..]).is_some(), - true, - "Actual response:\n{}", - String::from_utf8(one_response).unwrap() - ); - assert_eq!( - index_of( - &another_response, - &b"This is an index url which gives an overview of the different test urls available." - [..], - ) - .is_some(), - true, - "Actual response:\n{}", - String::from_utf8(another_response).unwrap() - ); + { + assert_eq!( + index_of( + &one_response, + &b"Education/Reference website with a low risk reputation score"[..] + ) + .is_some(), + true, + "Actual response:\n{}", + String::from_utf8(one_response).unwrap() + ); + assert_eq!( + index_of( + &one_response, + &b"This page simply displays this text without any specific content on it, it is just for testing purpose." + [..], + ) + .is_some(), + true, + "Actual response:\n{}", + String::from_utf8(one_response).unwrap() + ); + } + { + assert_eq!( + index_of( + &another_response, + &b"Public Information website with a low risk reputation score"[..] + ) + .is_some(), + true, + "Actual response:\n{}", + String::from_utf8(another_response).unwrap() + ); + assert_eq!( + index_of( + &another_response, + &b"This page simply displays this text without any specific content on it, it is just for testing purpose." + [..], + ) + .is_some(), + true, + "Actual response:\n{}", + String::from_utf8(another_response).unwrap() + ); + } } diff --git a/multinode_integration_tests/tests/min_hops_tests.rs b/multinode_integration_tests/tests/min_hops_tests.rs index ed0ef34f4..18efabd4f 100644 --- a/multinode_integration_tests/tests/min_hops_tests.rs +++ b/multinode_integration_tests/tests/min_hops_tests.rs @@ -30,7 +30,7 @@ fn assert_http_end_to_end_routing(min_hops: Hops) { let first_node = cluster.start_real_node(config); // For 1-hop route, 3 nodes are necessary if we use last node as the originating node - let nodes_count = (min_hops as usize) + 2; + let nodes_count = (min_hops as usize) * 2 + 1; let nodes = (0..nodes_count) .map(|i| { cluster.start_real_node( @@ -43,15 +43,15 @@ fn assert_http_end_to_end_routing(min_hops: Hops) { }) .collect::>(); - thread::sleep(Duration::from_millis(500 * (nodes.len() as u64))); + thread::sleep(Duration::from_millis(4000 * (nodes.len() as u64))); let last_node = nodes.last().unwrap(); let mut client = last_node.make_client(8080, 5000); - client.send_chunk(b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"); + client.send_chunk(b"GET /index.html HTTP/1.1\r\nHost: www.testingmcafeesites.com\r\n\r\n"); let response = client.wait_for_chunk(); assert_eq!( - index_of(&response, &b"

Example Domain

"[..]).is_some(), + index_of(&response, &b"URL for testing."[..]).is_some(), true, "Actual response:\n{}", String::from_utf8(response).unwrap() @@ -84,14 +84,14 @@ fn min_hops_can_be_changed_during_runtime() { thread::sleep(Duration::from_millis(1000)); let mut client = first_node.make_client(8080, 5000); - client.send_chunk(b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"); + client.send_chunk(b"GET /index.html HTTP/1.1\r\nHost: www.testingmcafeesites.com\r\n\r\n"); let response = client.wait_for_chunk(); // Client shutdown is necessary to re-initialize stream keys for old requests client.shutdown(); assert_eq!( - index_of(&response, &b"

Example Domain

"[..]).is_some(), + index_of(&response, &b"URL for testing."[..]).is_some(), true, "Actual response:\n{}", String::from_utf8(response).unwrap() @@ -108,12 +108,12 @@ fn min_hops_can_be_changed_during_runtime() { assert!(response.payload.is_ok()); let mut client = first_node.make_client(8080, 5000); - client.send_chunk(b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"); + client.send_chunk(b"GET /index.html HTTP/1.1\r\nHost: www.testingmcafeesites.com\r\n\r\n"); let response = client.wait_for_chunk(); assert_eq!( index_of( &response, - &b"

Subtitle: Can't find a route to www.example.com

"[..] + &b"

Subtitle: Can't find a route to www.testingmcafeesites.com

"[..] ) .is_some(), true, diff --git a/multinode_integration_tests/tests/neighbor_selection_test.rs b/multinode_integration_tests/tests/neighbor_selection_test.rs index 225713ba5..0ff393467 100644 --- a/multinode_integration_tests/tests/neighbor_selection_test.rs +++ b/multinode_integration_tests/tests/neighbor_selection_test.rs @@ -21,7 +21,7 @@ use std::convert::TryInto; use std::time::Duration; #[test] -fn debut_target_does_not_introduce_known_neighbors() { +fn if_debuter_already_knows_all_of_recipients_neighbors_recipient_redebuts() { let mut cluster = MASQNodeCluster::start().unwrap(); let one_common_neighbor = make_node_record(1234, true); let another_common_neighbor = make_node_record(2435, true); @@ -74,11 +74,7 @@ fn debut_target_does_not_introduce_known_neighbors() { let standard_gossip = Standard::from(&agrs); assert_eq!( standard_gossip.key_set(), - vec_to_set(vec![ - subject_real_node.main_public_key().clone(), - one_common_neighbor.public_key().clone(), - another_common_neighbor.public_key().clone(), - ]) + vec_to_set(vec![subject_real_node.main_public_key().clone(),]) ); } diff --git a/multinode_integration_tests/tests/self_test.rs b/multinode_integration_tests/tests/self_test.rs index 973affaf0..c19db4c33 100644 --- a/multinode_integration_tests/tests/self_test.rs +++ b/multinode_integration_tests/tests/self_test.rs @@ -16,6 +16,7 @@ use node_lib::sub_lib::dispatcher::Component; use node_lib::sub_lib::hopper::IncipientCoresPackage; use node_lib::sub_lib::route::Route; use node_lib::sub_lib::route::RouteSegment; +use node_lib::sub_lib::stream_key::StreamKey; use node_lib::test_utils::{make_meaningless_message_type, make_paying_wallet}; use std::collections::HashSet; use std::io::ErrorKind; @@ -68,6 +69,7 @@ fn server_relays_cores_package() { let masquerader = JsonMasquerader::new(); let server = MASQCoresServer::new(cluster.chain); let cryptde = server.main_cryptde(); + let stream_key = StreamKey::make_meaningless_stream_key(); let mut client = MASQCoresClient::new(server.local_addr(), cryptde); let mut route = Route::one_way( RouteSegment::new( @@ -82,7 +84,7 @@ fn server_relays_cores_package() { let incipient = IncipientCoresPackage::new( cryptde, route.clone(), - make_meaningless_message_type(), + make_meaningless_message_type(stream_key), &cryptde.public_key(), ) .unwrap(); @@ -99,7 +101,7 @@ fn server_relays_cores_package() { route.shift(cryptde).unwrap(); assert_eq!(expired.remaining_route, route); - assert_eq!(expired.payload, make_meaningless_message_type()); + assert_eq!(expired.payload, make_meaningless_message_type(stream_key)); } #[test] @@ -111,6 +113,7 @@ fn one_mock_node_talks_to_another() { let mock_node_1 = cluster.get_mock_node_by_name("mock_node_1").unwrap(); let mock_node_2 = cluster.get_mock_node_by_name("mock_node_2").unwrap(); let cryptde = CryptDENull::new(TEST_DEFAULT_CHAIN); + let stream_key = StreamKey::make_meaningless_stream_key(); let route = Route::one_way( RouteSegment::new( vec![ @@ -127,7 +130,7 @@ fn one_mock_node_talks_to_another() { let incipient_cores_package = IncipientCoresPackage::new( &cryptde, route, - make_meaningless_message_type(), + make_meaningless_message_type(stream_key), &mock_node_2.main_public_key(), ) .unwrap(); @@ -156,7 +159,7 @@ fn one_mock_node_talks_to_another() { assert_eq!(package_to, mock_node_2.socket_addr(PortSelector::First)); assert_eq!( expired_cores_package.payload, - make_meaningless_message_type() + make_meaningless_message_type(stream_key) ); } diff --git a/multinode_integration_tests/tests/verify_bill_payment.rs b/multinode_integration_tests/tests/verify_bill_payment.rs index d421f82b2..e5fddc67f 100644 --- a/multinode_integration_tests/tests/verify_bill_payment.rs +++ b/multinode_integration_tests/tests/verify_bill_payment.rs @@ -465,7 +465,7 @@ fn verify_pending_payables() { } MASQNodeUtils::assert_node_wrote_log_containing( real_consuming_node.name(), - "Found 3 pending payables and 0 unfinalized failures to process", + "Found 3 pending payables and 0 suspected failures to process", Duration::from_secs(5), ); MASQNodeUtils::assert_node_wrote_log_containing( diff --git a/node/.gitignore b/node/.gitignore index 4bdd95441..341f40dbf 100644 --- a/node/.gitignore +++ b/node/.gitignore @@ -73,6 +73,9 @@ cmake-build-debug/ # IntelliJ out/ +# Local IDE database configuration +.idea/db-forest-config.xml + # mpeltonen/sbt-idea plugin .idea_modules/ diff --git a/node/Cargo.lock b/node/Cargo.lock index c825fda4b..443b6a132 100644 --- a/node/Cargo.lock +++ b/node/Cargo.lock @@ -182,7 +182,7 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "automap" -version = "0.9.0" +version = "0.9.1" dependencies = [ "crossbeam-channel 0.5.1", "flexi_logger 0.17.1", @@ -1578,7 +1578,7 @@ dependencies = [ [[package]] name = "ip_country" -version = "0.1.0" +version = "0.9.1" dependencies = [ "csv", "ipnetwork", @@ -1868,7 +1868,7 @@ dependencies = [ [[package]] name = "masq" -version = "0.9.0" +version = "0.9.1" dependencies = [ "atty", "clap", @@ -1889,7 +1889,7 @@ dependencies = [ [[package]] name = "masq_lib" -version = "0.9.0" +version = "0.9.1" dependencies = [ "actix", "clap", @@ -2082,7 +2082,7 @@ dependencies = [ [[package]] name = "multinode_integration_tests" -version = "0.9.0" +version = "0.9.1" dependencies = [ "base64 0.13.0", "crossbeam-channel 0.5.1", @@ -2176,7 +2176,7 @@ dependencies = [ [[package]] name = "node" -version = "0.9.0" +version = "0.9.1" dependencies = [ "actix", "automap", @@ -3750,7 +3750,7 @@ dependencies = [ [[package]] name = "test_utilities" -version = "0.1.0" +version = "0.9.1" [[package]] name = "textwrap" diff --git a/node/Cargo.toml b/node/Cargo.toml index 80d75b5ef..cbcb8ac2d 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node" -version = "0.9.0" +version = "0.9.1" license = "GPL-3.0-only" authors = ["Dan Wiebe ", "MASQ"] description = "MASQ Node is the foundation of MASQ Network, an open-source network that allows anyone to allocate spare computing resources to make the internet a free and fair place for the entire world." @@ -15,7 +15,7 @@ automap = { path = "../automap"} backtrace = "0.3.57" base64 = "0.13.0" bytes = "0.4.12" -time = {version = "0.3.11", features = [ "macros", "parsing" ]} +time = { version = "0.3.11", features = ["macros", "local-offset", "parsing"] } clap = "2.33.3" crossbeam-channel = "0.5.1" dirs = "4.0.0" diff --git a/node/src/accountant/db_access_objects/failed_payable_dao.rs b/node/src/accountant/db_access_objects/failed_payable_dao.rs index 2b83fdfab..be875807e 100644 --- a/node/src/accountant/db_access_objects/failed_payable_dao.rs +++ b/node/src/accountant/db_access_objects/failed_payable_dao.rs @@ -163,7 +163,7 @@ impl Display for FailureRetrieveCondition { ) } FailureRetrieveCondition::EveryRecheckRequiredRecord => { - write!(f, "WHERE status LIKE 'RecheckRequired%'") + write!(f, "WHERE status LIKE '%RecheckRequired%'") } } } @@ -657,9 +657,14 @@ mod tests { .to_string(), "WHERE status = '{\"RecheckRequired\":\"Waiting\"}'" ); + assert_eq!( + FailureRetrieveCondition::ByReceiverAddresses(BTreeSet::from([make_address(1), make_address(2)])) + .to_string(), + "WHERE receiver_address IN ('0x0000000000000000000003000000000003000000', '0x0000000000000000000006000000000006000000')" + ); assert_eq!( FailureRetrieveCondition::EveryRecheckRequiredRecord.to_string(), - "WHERE status LIKE 'RecheckRequired%'" + "WHERE status LIKE '%RecheckRequired%'" ); } @@ -739,19 +744,6 @@ mod tests { ); } - #[test] - fn retrieve_condition_display_works() { - assert_eq!( - FailureRetrieveCondition::ByStatus(RetryRequired).to_string(), - "WHERE status = '\"RetryRequired\"'" - ); - assert_eq!( - FailureRetrieveCondition::ByReceiverAddresses(BTreeSet::from([make_address(1), make_address(2)])) - .to_string(), - "WHERE receiver_address IN ('0x0000000000000000000003000000000003000000', '0x0000000000000000000006000000000006000000')" - ) - } - #[test] fn can_retrieve_all_txs_ordered_by_timestamp_and_nonce() { let home_dir = ensure_node_home_directory_exists( @@ -904,6 +896,63 @@ mod tests { assert!(!result.contains(&tx4)); } + #[test] + fn can_retrieve_every_recheck_required() { + let home_dir = ensure_node_home_directory_exists( + "failed_payable_dao", + "can_retrieve_every_recheck_required", + ); + let wrapped_conn = DbInitializerReal::default() + .initialize(&home_dir, DbInitializationConfig::test_default()) + .unwrap(); + let subject = FailedPayableDaoReal::new(wrapped_conn); + let address1 = make_address(1); + let address2 = make_address(2); + let address3 = make_address(3); + let address4 = make_address(4); + let tx1 = FailedTxBuilder::default() + .hash(make_tx_hash(1)) + .receiver_address(address1) + .nonce(1) + .status(RetryRequired) + .build(); + let tx2 = FailedTxBuilder::default() + .hash(make_tx_hash(2)) + .receiver_address(address2) + .nonce(2) + .status(RecheckRequired(ValidationStatus::Waiting)) + .build(); + let tx3 = FailedTxBuilder::default() + .hash(make_tx_hash(3)) + .receiver_address(address3) + .nonce(3) + .status(Concluded) + .build(); + let tx4 = FailedTxBuilder::default() + .hash(make_tx_hash(4)) + .receiver_address(address4) + .nonce(4) + .status(RecheckRequired(ValidationStatus::Reattempting( + PreviousAttempts::new( + BlockchainErrorKind::AppRpc(AppRpcErrorKind::Remote( + RemoteErrorKind::Unreachable, + )), + &SimpleClockReal::default(), + ), + ))) + .build(); + subject + .insert_new_records(&BTreeSet::from([tx1, tx2.clone(), tx3, tx4.clone()])) + .unwrap(); + + let result = + subject.retrieve_txs(Some(FailureRetrieveCondition::EveryRecheckRequiredRecord)); + + assert_eq!(result.len(), 2); + assert!(result.contains(&tx2)); + assert!(result.contains(&tx4)); + } + #[test] fn update_statuses_works() { let home_dir = diff --git a/node/src/accountant/mod.rs b/node/src/accountant/mod.rs index 153158dd4..a82d39887 100644 --- a/node/src/accountant/mod.rs +++ b/node/src/accountant/mod.rs @@ -30,7 +30,7 @@ use crate::accountant::scanners::pending_payable_scanner::utils::{ PendingPayableScanResult, TxHashByTable, }; use crate::accountant::scanners::scan_schedulers::{ - PayableSequenceScanner, ScanReschedulingAfterEarlyStop, ScanSchedulers, + ScanReschedulingAfterEarlyStop, ScanSchedulers, UnableToStartScanner, }; use crate::accountant::scanners::{Scanners, StartScanError}; use crate::blockchain::blockchain_bridge::{ @@ -312,11 +312,25 @@ impl Handler for Accountant { impl Handler for Accountant { type Result = (); - fn handle(&mut self, msg: ScanForReceivables, ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: ScanForReceivables, _ctx: &mut Self::Context) -> Self::Result { // By now we know it is an automatic scan. The ReceivableScanner is independent of other // scanners and rescheduled regularly, just here. - self.handle_request_of_scan_for_receivable(msg.response_skeleton_opt); - self.scan_schedulers.receivable.schedule(ctx, &self.logger); + let scheduling_hint = self.handle_request_of_scan_for_receivable(msg.response_skeleton_opt); + + match scheduling_hint { + ScanReschedulingAfterEarlyStop::Schedule(other_scan_type) => unreachable!( + "Early stopped receivable scan was suggested to be followed up by the scan \ + for {:?}, which is not supported though", + other_scan_type + ), + ScanReschedulingAfterEarlyStop::DoNotSchedule => { + trace!( + self.logger, + "No early rescheduling, as the receivable scan did find results, or this \ + is the NullScanner" + ) + } + } } } @@ -348,8 +362,8 @@ impl Handler for Accountant { if let Some(node_to_ui_msg) = ui_msg_opt { info!( self.logger, - "Re-running the pending payable scan is recommended, as some \ - parts did not finish last time." + "Re-running the pending payable scan is recommended, as some parts \ + did not finish last time." ); self.ui_message_sub_opt .as_ref() @@ -401,13 +415,19 @@ impl Handler for Accountant { impl Handler for Accountant { type Result = (); - fn handle(&mut self, msg: ReceivedPayments, _ctx: &mut Self::Context) -> Self::Result { - if let Some(node_to_ui_msg) = self.scanners.finish_receivable_scan(msg, &self.logger) { - self.ui_message_sub_opt - .as_ref() - .expect("UIGateway is not bound") - .try_send(node_to_ui_msg) - .expect("UIGateway is dead"); + fn handle(&mut self, msg: ReceivedPayments, ctx: &mut Self::Context) -> Self::Result { + match self.scanners.finish_receivable_scan(msg, &self.logger) { + None => self.scan_schedulers.receivable.schedule(ctx, &self.logger), + Some(node_to_ui_msg) => { + self.ui_message_sub_opt + .as_ref() + .expect("UIGateway is not bound") + .try_send(node_to_ui_msg) + .expect("UIGateway is dead"); + // Externally triggered scans are not allowed to provoke an unwinding scan sequence + // with intervals. The only exception is the PendingPayableScanner that is always + // followed by the retry-payable scanner in a tight tandem. + } } } } @@ -417,27 +437,34 @@ impl Handler for Accountant { fn handle(&mut self, scan_error: ScanError, ctx: &mut Self::Context) -> Self::Result { error!(self.logger, "Received ScanError: {:?}", scan_error); + self.scanners .acknowledge_scan_error(&scan_error, &self.logger); match scan_error.response_skeleton_opt { - None => match scan_error.scan_type { - DetailedScanType::NewPayables => self - .scan_schedulers - .payable - .schedule_new_payable_scan(ctx, &self.logger), - DetailedScanType::RetryPayables => self - .scan_schedulers - .payable - .schedule_retry_payable_scan(ctx, None, &self.logger), - DetailedScanType::PendingPayables => self - .scan_schedulers - .pending_payable - .schedule(ctx, &self.logger), - DetailedScanType::Receivables => { - self.scan_schedulers.receivable.schedule(ctx, &self.logger) + None => { + debug!( + self.logger, + "Trying to restore the scan train after a crash" + ); + match scan_error.scan_type { + DetailedScanType::NewPayables => self + .scan_schedulers + .payable + .schedule_new_payable_scan(ctx, &self.logger), + DetailedScanType::RetryPayables => self + .scan_schedulers + .payable + .schedule_retry_payable_scan(ctx, None, &self.logger), + DetailedScanType::PendingPayables => self + .scan_schedulers + .pending_payable + .schedule(ctx, &self.logger), + DetailedScanType::Receivables => { + self.scan_schedulers.receivable.schedule(ctx, &self.logger) + } } - }, + } Some(response_skeleton) => { let error_msg = NodeToUiMessage { target: ClientId(response_skeleton.client_id), @@ -613,12 +640,11 @@ impl Accountant { &self, service_rate: u64, byte_rate: u64, + total_charge: u128, timestamp: SystemTime, payload_size: usize, wallet: &Wallet, ) { - let byte_charge = byte_rate as u128 * (payload_size as u128); - let total_charge = service_rate as u128 + byte_charge; if !self.our_wallet(wallet) { match self.receivable_dao .as_ref() @@ -647,12 +673,11 @@ impl Accountant { &self, service_rate: u64, byte_rate: u64, + total_charge: u128, timestamp: SystemTime, payload_size: usize, wallet: &Wallet, ) { - let byte_charge = byte_rate as u128 * (payload_size as u128); - let total_charge = service_rate as u128 + byte_charge; if !self.our_wallet(wallet) { match self.payable_dao .as_ref() @@ -732,15 +757,18 @@ impl Accountant { &mut self, msg: ReportRoutingServiceProvidedMessage, ) { + let total_charge = Self::total_charge(msg.byte_rate, msg.service_rate, msg.payload_size); trace!( self.logger, - "Charging routing of {} bytes to wallet {}", + "Charging {} wei for routing of {} bytes to wallet {}", + total_charge, msg.payload_size, msg.paying_wallet ); self.record_service_provided( msg.service_rate, msg.byte_rate, + total_charge, msg.timestamp, msg.payload_size, &msg.paying_wallet, @@ -751,9 +779,11 @@ impl Accountant { &mut self, msg: ReportExitServiceProvidedMessage, ) { + let total_charge = Self::total_charge(msg.byte_rate, msg.service_rate, msg.payload_size); trace!( self.logger, - "Charging exit service for {} bytes to wallet {} at {} per service and {} per byte", + "Charging {} wei for exit service for {} bytes to wallet {} at {} per service and {} per byte", + total_charge, msg.payload_size, msg.paying_wallet, msg.service_rate, @@ -762,6 +792,7 @@ impl Accountant { self.record_service_provided( msg.service_rate, msg.byte_rate, + total_charge, msg.timestamp, msg.payload_size, &msg.paying_wallet, @@ -778,31 +809,45 @@ impl Accountant { fn handle_report_services_consumed_message(&mut self, msg: ReportServicesConsumedMessage) { let msg_id = self.msg_id(); + let total_charge = Self::total_charge( + msg.exit.byte_rate, + msg.exit.service_rate, + msg.exit.payload_size, + ); trace!( self.logger, - "MsgId {}: Accruing debt to {} for consuming {} exited bytes", + "MsgId {}: Accruing {} wei of debt to {} for consuming {} exited bytes", msg_id, + total_charge, msg.exit.earning_wallet, msg.exit.payload_size ); self.record_service_consumed( msg.exit.service_rate, msg.exit.byte_rate, + total_charge, msg.timestamp, msg.exit.payload_size, &msg.exit.earning_wallet, ); msg.routing.iter().for_each(|routing_service| { + let total_charge = Self::total_charge( + routing_service.byte_rate, + routing_service.service_rate, + msg.routing_payload_size, + ); trace!( self.logger, - "MsgId {}: Accruing debt to {} for consuming {} routed bytes", + "MsgId {}: Accruing {} wei of debt to {} for consuming {} routed bytes", msg_id, + total_charge, routing_service.earning_wallet, msg.routing_payload_size ); self.record_service_consumed( routing_service.service_rate, routing_service.byte_rate, + total_charge, msg.timestamp, msg.routing_payload_size, &routing_service.earning_wallet, @@ -975,7 +1020,7 @@ impl Accountant { None => Err(StartScanError::NoConsumingWalletFound), }; - self.scan_schedulers.payable.reset_scan_timer(); + self.scan_schedulers.payable.reset_scan_timer(&self.logger); match result { Ok(scan_message) => { @@ -986,8 +1031,8 @@ impl Accountant { .expect("BlockchainBridge is dead"); ScanReschedulingAfterEarlyStop::DoNotSchedule } - Err(e) => self.handle_start_scan_error_and_prevent_scan_stall_point( - PayableSequenceScanner::NewPayables, + Err(e) => self.handle_start_scan_error( + UnableToStartScanner::NewPayables, e, response_skeleton_opt, ), @@ -1018,10 +1063,9 @@ impl Accountant { .expect("BlockchainBridge is dead"); } Err(e) => { - // It is thrown away and there is no rescheduling downstream because every error - // happening here on the start resolves into a panic by the current design - let _ = self.handle_start_scan_error_and_prevent_scan_stall_point( - PayableSequenceScanner::RetryPayables, + // Any error here panics by design, so the return value is unreachable/ignored. + let _ = self.handle_start_scan_error( + UnableToStartScanner::RetryPayables, e, response_skeleton_opt, ); @@ -1056,8 +1100,8 @@ impl Accountant { } Err(e) => { let initial_pending_payable_scan = self.scanners.initial_pending_payable_scan(); - self.handle_start_scan_error_and_prevent_scan_stall_point( - PayableSequenceScanner::PendingPayables { + self.handle_start_scan_error( + UnableToStartScanner::PendingPayables { initial_pending_payable_scan, }, e, @@ -1073,9 +1117,9 @@ impl Accountant { hint } - fn handle_start_scan_error_and_prevent_scan_stall_point( + fn handle_start_scan_error( &self, - scanner: PayableSequenceScanner, + scanner: UnableToStartScanner, e: StartScanError, response_skeleton_opt: Option, ) -> ScanReschedulingAfterEarlyStop { @@ -1102,7 +1146,7 @@ impl Accountant { fn handle_request_of_scan_for_receivable( &mut self, response_skeleton_opt: Option, - ) { + ) -> ScanReschedulingAfterEarlyStop { let result: Result = self.scanners.start_receivable_scan_guarded( &self.earning_wallet, @@ -1113,29 +1157,22 @@ impl Accountant { ); match result { - Ok(scan_message) => self - .retrieve_transactions_sub_opt - .as_ref() - .expect("BlockchainBridge is unbound") - .try_send(scan_message) - .expect("BlockchainBridge is dead"), - Err(e) => { - e.log_error( - &self.logger, - ScanType::Receivables, - response_skeleton_opt.is_some(), - ); - - if let Some(skeleton) = response_skeleton_opt { - self.ui_message_sub_opt - .as_ref() - .expect("UiGateway is unbound") - .try_send(NodeToUiMessage { - target: MessageTarget::ClientId(skeleton.client_id), - body: UiScanResponse {}.tmb(skeleton.context_id), - }) - .expect("UiGateway is dead"); - }; + Ok(scan_message) => { + self.retrieve_transactions_sub_opt + .as_ref() + .expect("BlockchainBridge is unbound") + .try_send(scan_message) + .expect("BlockchainBridge is dead"); + ScanReschedulingAfterEarlyStop::DoNotSchedule + } + Err(e) => + // Any error here panics by design, so the return value is unreachable/ignored. + { + self.handle_start_scan_error( + UnableToStartScanner::Receivables, + e, + response_skeleton_opt, + ) } } } @@ -1205,6 +1242,11 @@ impl Accountant { } } + fn total_charge(byte_rate: u64, service_rate: u64, payload_size: usize) -> u128 { + let byte_charge = byte_rate as u128 * (payload_size as u128); + service_rate as u128 + byte_charge + } + fn financial_statistics(&self) -> Ref<'_, FinancialStatistics> { self.financial_statistics.borrow() } @@ -2037,8 +2079,8 @@ mod tests { let system = System::new("test"); subject.scan_schedulers.automatic_scans_enabled = false; // Making sure we would kill the test if any sort of scan was scheduled - subject.scan_schedulers.payable.retry_payable_notify = - Box::new(NotifyHandleMock::default().panic_on_schedule_attempt()); + subject.scan_schedulers.payable.retry_payable_notify_later = + Box::new(NotifyLaterHandleMock::default().panic_on_schedule_attempt()); subject.scan_schedulers.payable.new_payable_notify_later = Box::new(NotifyLaterHandleMock::default().panic_on_schedule_attempt()); subject.scan_schedulers.payable.new_payable_notify = @@ -2274,6 +2316,7 @@ mod tests { ]) .build(); subject.scan_schedulers.automatic_scans_enabled = false; + subject.scan_schedulers.payable.retry_payable_scan_interval = Duration::from_millis(1); let (blockchain_bridge, _, blockchain_bridge_recording_arc) = make_recorder(); let (ui_gateway, _, ui_gateway_recording_arc) = make_recorder(); let ui_gateway = @@ -2281,7 +2324,7 @@ mod tests { let (peer_actors, peer_addresses) = peer_actors_builder() .blockchain_bridge(blockchain_bridge) .ui_gateway(ui_gateway) - .build_and_provide_addresses(); + .build_with_addresses(); let subject_addr = subject.start(); let system = System::new("test"); let response_skeleton_opt = Some(ResponseSkeleton { @@ -2715,6 +2758,40 @@ mod tests { ); } + #[test] + #[should_panic( + expected = "internal error: entered unreachable code: Early stopped receivable scan \ + was suggested to be followed up by the scan for PendingPayables, which is not supported though" + )] + fn start_scan_error_in_receivables_and_unexpected_instruction_from_early_stop_scan_rescheduling( + ) { + let mut subject = AccountantBuilder::default().build(); + let reschedule_on_error_resolver = RescheduleScanOnErrorResolverMock::default() + .resolve_rescheduling_on_error_result(ScanReschedulingAfterEarlyStop::Schedule( + ScanType::PendingPayables, + )); + let receivable_scanner = ScannerMock::default() + .scan_started_at_result(None) + .start_scan_result(Err(StartScanError::NoConsumingWalletFound)); + subject + .scanners + .replace_scanner(ScannerReplacement::Receivable(ReplacementType::Mock( + receivable_scanner, + ))); + subject.scan_schedulers.reschedule_on_error_resolver = + Box::new(reschedule_on_error_resolver); + let system = System::new("test"); + let subject_addr = subject.start(); + + subject_addr + .try_send(ScanForReceivables { + response_skeleton_opt: None, + }) + .unwrap(); + + system.run(); + } + #[test] fn received_payments_with_response_skeleton_sends_response_to_ui_gateway() { let mut config = bc_from_earning_wallet(make_wallet("earning_wallet")); @@ -2724,7 +2801,7 @@ mod tests { receivable_scan_interval: Duration::from_millis(10_000), }); config.automatic_scans_enabled = false; - let subject = AccountantBuilder::default() + let mut subject = AccountantBuilder::default() .bootstrapper_config(config) .config_dao( ConfigDaoMock::new() @@ -2732,6 +2809,9 @@ mod tests { .set_result(Ok(())), ) .build(); + // Another scan must not be scheduled + subject.scan_schedulers.receivable.handle = + Box::new(NotifyLaterHandleMock::default().panic_on_schedule_attempt()); let (ui_gateway, _, ui_gateway_recording_arc) = make_recorder(); let subject_addr = subject.start(); let system = System::new("test"); @@ -2838,11 +2918,13 @@ mod tests { let test_name = "accountant_scans_after_startup_and_does_not_detect_any_pending_payables"; let scan_params = ScanParams::default(); let notify_and_notify_later_params = NotifyAndNotifyLaterParams::default(); - let time_until_next_scan_params_arc = Arc::new(Mutex::new(vec![])); + let compute_time_to_next_scan_params_arc = Arc::new(Mutex::new(vec![])); let earning_wallet = make_wallet("earning"); let consuming_wallet = make_wallet("consuming"); let system = System::new(test_name); - let _ = SystemKillerActor::new(Duration::from_secs(10)).start(); + let (blockchain_bridge, _, _) = make_recorder(); + let blockchain_bridge = blockchain_bridge + .system_stop_conditions(match_lazily_every_type_id!(RetrieveTransactions)); let config = bc_from_wallets(consuming_wallet.clone(), earning_wallet.clone()); let payable_scanner = ScannerMock::new() .scan_started_at_result(None) @@ -2855,25 +2937,31 @@ mod tests { let receivable_scanner = ScannerMock::new() .scan_started_at_result(None) .start_scan_params(&scan_params.receivable_start_scan) - .start_scan_result(Err(StartScanError::NothingToProcess)); - let (subject, new_payable_expected_computed_interval, receivable_scan_interval) = + .start_scan_result(Ok(RetrieveTransactions { + recipient: earning_wallet.clone(), + response_skeleton_opt: None, + })); + let (subject, new_payable_expected_computed_interval) = configure_accountant_for_startup_with_preexisting_pending_payables( test_name, ¬ify_and_notify_later_params, - &time_until_next_scan_params_arc, + &compute_time_to_next_scan_params_arc, config, pending_payable_scanner, receivable_scanner, payable_scanner, ); - let peer_actors = peer_actors_builder().build(); + let peer_actors = peer_actors_builder() + .blockchain_bridge(blockchain_bridge) + .build(); let subject_addr: Addr = subject.start(); let subject_subs = Accountant::make_subs_from(&subject_addr); send_bind_message!(subject_subs, peer_actors); send_start_message!(subject_subs); - // The system is stopped by the NotifyLaterHandleMock for the Receivable scanner + // The system is stopped by the time the RetrieveTransactions msg arrives at the mocked + // BlockchainBridge. let before = SystemTime::now(); system.run(); let after = SystemTime::now(); @@ -2888,15 +2976,13 @@ mod tests { assert_payable_scanner_for_no_pending_payable_found( &scan_params.payable_start_scan, ¬ify_and_notify_later_params, - time_until_next_scan_params_arc, + compute_time_to_next_scan_params_arc, new_payable_expected_computed_interval, ); assert_receivable_scanner( test_name, earning_wallet, &scan_params.receivable_start_scan, - ¬ify_and_notify_later_params.receivables_notify_later, - receivable_scan_interval, ); // The test lays down evidences that the NewPayableScanner couldn't run before // the PendingPayableScanner, which is an intention. @@ -2954,8 +3040,11 @@ mod tests { let receivable_scanner = ScannerMock::new() .scan_started_at_result(None) .start_scan_params(&scan_params.receivable_start_scan) - .start_scan_result(Err(StartScanError::NothingToProcess)); - let (subject, expected_pending_payable_notify_later_interval, receivable_scan_interval) = + .start_scan_result(Ok(RetrieveTransactions { + recipient: earning_wallet.clone(), + response_skeleton_opt: None, + })); + let (subject, expected_pending_payable_notify_later_interval) = configure_accountant_for_startup_with_no_preexisting_pending_payables( test_name, ¬ify_and_notify_later_params, @@ -2964,7 +3053,7 @@ mod tests { pending_payable_scanner, receivable_scanner, ); - let (peer_actors, addresses) = peer_actors_builder().build_and_provide_addresses(); + let (peer_actors, addresses) = peer_actors_builder().build_with_addresses(); let subject_addr: Addr = subject.start(); let subject_subs = Accountant::make_subs_from(&subject_addr); let expected_tx_receipts_msg = TxReceiptsMessage { @@ -3031,8 +3120,6 @@ mod tests { test_name, earning_wallet, &scan_params.receivable_start_scan, - ¬ify_and_notify_later_params.receivables_notify_later, - receivable_scan_interval, ); // Since the assertions proved that the pending payable scanner had run multiple times // before the new payable scanner started or was scheduled, the front position definitely @@ -3055,15 +3142,14 @@ mod tests { struct NotifyAndNotifyLaterParams { new_payables_notify_later: Arc>>, new_payables_notify: Arc>>, - retry_payables_notify: Arc>>, + retry_payables_notify_later: Arc>>, pending_payables_notify_later: Arc>>, - receivables_notify_later: Arc>>, } fn configure_accountant_for_startup_with_preexisting_pending_payables( test_name: &str, notify_and_notify_later_params: &NotifyAndNotifyLaterParams, - time_until_next_scan_params_arc: &Arc>>, + compute_time_to_next_scan_params_arc: &Arc>>, config: BootstrapperConfig, pending_payable_scanner: ScannerMock< RequestTransactionReceipts, @@ -3076,7 +3162,7 @@ mod tests { Option, >, payable_scanner: ScannerMock, - ) -> (Accountant, Duration, Duration) { + ) -> (Accountant, Duration) { let mut subject = make_subject_and_inject_scanners( test_name, config, @@ -3086,7 +3172,6 @@ mod tests { ); let new_payable_expected_computed_interval = Duration::from_secs(3600); // Important that this is made short because the test relies on it with the system stop. - let receivable_scan_interval = Duration::from_millis(50); subject.scan_schedulers.pending_payable.handle = Box::new( NotifyLaterHandleMock::default() .notify_later_params(¬ify_and_notify_later_params.pending_payables_notify_later), @@ -3095,30 +3180,21 @@ mod tests { NotifyLaterHandleMock::default() .notify_later_params(¬ify_and_notify_later_params.new_payables_notify_later), ); - subject.scan_schedulers.payable.retry_payable_notify = Box::new( - NotifyHandleMock::default() - .notify_params(¬ify_and_notify_later_params.retry_payables_notify), + subject.scan_schedulers.payable.retry_payable_notify_later = Box::new( + NotifyLaterHandleMock::default() + .notify_later_params(¬ify_and_notify_later_params.retry_payables_notify_later), ); subject.scan_schedulers.payable.new_payable_notify = Box::new( NotifyHandleMock::default() .notify_params(¬ify_and_notify_later_params.new_payables_notify), ); - let receivable_notify_later_handle_mock = NotifyLaterHandleMock::default() - .notify_later_params(¬ify_and_notify_later_params.receivables_notify_later) - .stop_system_on_count_received(1); - subject.scan_schedulers.receivable.handle = Box::new(receivable_notify_later_handle_mock); - subject.scan_schedulers.receivable.interval = receivable_scan_interval; let interval_computer = NewPayableScanIntervalComputerMock::default() - .time_until_next_scan_params(&time_until_next_scan_params_arc) - .time_until_next_scan_result(ScanTiming::WaitFor( + .compute_time_to_next_scan_params(&compute_time_to_next_scan_params_arc) + .compute_time_to_next_scan_result(ScanTiming::WaitFor( new_payable_expected_computed_interval, )); subject.scan_schedulers.payable.interval_computer = Box::new(interval_computer); - ( - subject, - new_payable_expected_computed_interval, - receivable_scan_interval, - ) + (subject, new_payable_expected_computed_interval) } fn configure_accountant_for_startup_with_no_preexisting_pending_payables( @@ -3136,7 +3212,7 @@ mod tests { ReceivedPayments, Option, >, - ) -> (Accountant, Duration, Duration) { + ) -> (Accountant, Duration) { let mut subject = make_subject_and_inject_scanners( test_name, config, @@ -3144,37 +3220,30 @@ mod tests { receivable_scanner, payable_scanner, ); + let retry_payable_scan_interval = Duration::from_millis(1); let pending_payable_scan_interval = Duration::from_secs(3600); - let receivable_scan_interval = Duration::from_secs(3600); let pending_payable_notify_later_handle_mock = NotifyLaterHandleMock::default() .notify_later_params(¬ify_and_notify_later_params.pending_payables_notify_later) // This should stop the system .stop_system_on_count_received(1); subject.scan_schedulers.pending_payable.handle = Box::new(pending_payable_notify_later_handle_mock); + subject.scan_schedulers.payable.retry_payable_scan_interval = retry_payable_scan_interval; subject.scan_schedulers.pending_payable.interval = pending_payable_scan_interval; subject.scan_schedulers.payable.new_payable_notify_later = Box::new( NotifyLaterHandleMock::default() .notify_later_params(¬ify_and_notify_later_params.new_payables_notify_later), ); - subject.scan_schedulers.payable.retry_payable_notify = Box::new( - NotifyHandleMock::default() - .notify_params(¬ify_and_notify_later_params.retry_payables_notify) + subject.scan_schedulers.payable.retry_payable_notify_later = Box::new( + NotifyLaterHandleMock::default() + .notify_later_params(¬ify_and_notify_later_params.retry_payables_notify_later) .capture_msg_and_let_it_fly_on(), ); subject.scan_schedulers.payable.new_payable_notify = Box::new( NotifyHandleMock::default() .notify_params(¬ify_and_notify_later_params.new_payables_notify), ); - let receivable_notify_later_handle_mock = NotifyLaterHandleMock::default() - .notify_later_params(¬ify_and_notify_later_params.receivables_notify_later); - subject.scan_schedulers.receivable.interval = receivable_scan_interval; - subject.scan_schedulers.receivable.handle = Box::new(receivable_notify_later_handle_mock); - ( - subject, - pending_payable_scan_interval, - receivable_scan_interval, - ) + (subject, pending_payable_scan_interval) } fn make_subject_and_inject_scanners( @@ -3333,7 +3402,7 @@ mod tests { Mutex, Logger, String)>>, >, notify_and_notify_later_params: &NotifyAndNotifyLaterParams, - time_until_next_scan_until_next_new_payable_scan_params_arc: Arc>>, + time_until_next_new_payable_scan_params_arc: Arc>>, new_payable_expected_computed_interval: Duration, ) { // Note that there is no functionality from the payable scanner actually running. @@ -3351,14 +3420,9 @@ mod tests { new_payable_expected_computed_interval )] ); - let time_until_next_scan_until_next_new_payable_scan_params = - time_until_next_scan_until_next_new_payable_scan_params_arc - .lock() - .unwrap(); - assert_eq!( - *time_until_next_scan_until_next_new_payable_scan_params, - vec![()] - ); + let time_until_next_new_payable_scan_params = + time_until_next_new_payable_scan_params_arc.lock().unwrap(); + assert_eq!(*time_until_next_new_payable_scan_params, vec![()]); let payable_scanner_start_scan = payable_scanner_start_scan_arc.lock().unwrap(); assert!( payable_scanner_start_scan.is_empty(), @@ -3374,7 +3438,7 @@ mod tests { scan_for_new_payables_notify_params ); let scan_for_retry_payables_notify_params = notify_and_notify_later_params - .retry_payables_notify + .retry_payables_notify_later .lock() .unwrap(); assert!( @@ -3460,14 +3524,17 @@ mod tests { scan_for_new_payables_notify_params ); let scan_for_retry_payables_notify_params = notify_and_notify_later_params - .retry_payables_notify + .retry_payables_notify_later .lock() .unwrap(); assert_eq!( *scan_for_retry_payables_notify_params, - vec![ScanForRetryPayables { - response_skeleton_opt: None - }], + vec![( + ScanForRetryPayables { + response_skeleton_opt: None + }, + Duration::from_millis(1) + )], ); } @@ -3477,16 +3544,8 @@ mod tests { receivable_start_scan_params_arc: &Arc< Mutex, Logger, String)>>, >, - scan_for_receivables_notify_later_params_arc: &Arc< - Mutex>, - >, - receivable_scan_interval: Duration, ) { assert_receivable_scan_ran(test_name, receivable_start_scan_params_arc, earning_wallet); - assert_another_receivable_scan_scheduled( - scan_for_receivables_notify_later_params_arc, - receivable_scan_interval, - ) } fn assert_receivable_scan_ran( @@ -3514,25 +3573,6 @@ mod tests { assert_using_the_same_logger(&r_logger, test_name, Some("r")); } - fn assert_another_receivable_scan_scheduled( - scan_for_receivables_notify_later_params_arc: &Arc< - Mutex>, - >, - receivable_scan_interval: Duration, - ) { - let scan_for_receivables_notify_later_params = - scan_for_receivables_notify_later_params_arc.lock().unwrap(); - assert_eq!( - *scan_for_receivables_notify_later_params, - vec![( - ScanForReceivables { - response_skeleton_opt: None - }, - receivable_scan_interval - )] - ); - } - #[test] fn initial_pending_payable_scan_if_some_payables_found() { let sent_payable_dao = @@ -3629,23 +3669,31 @@ mod tests { let start_scan_params_arc = Arc::new(Mutex::new(vec![])); let notify_later_receivable_params_arc = Arc::new(Mutex::new(vec![])); let system = System::new(test_name); + let (blockchain_bridge, _, blockchain_bridge_recording_arc) = make_recorder(); + let blockchain_bridge = blockchain_bridge.system_stop_conditions( + match_lazily_every_type_id!(RetrieveTransactions, RetrieveTransactions), + ); SystemKillerActor::new(Duration::from_secs(10)).start(); // a safety net for GitHub Actions + let recipient = make_wallet("some_recipient"); let receivable_scanner = ScannerMock::new() .scan_started_at_result(None) .scan_started_at_result(None) .start_scan_params(&start_scan_params_arc) - .start_scan_result(Err(StartScanError::NothingToProcess)) .start_scan_result(Ok(RetrieveTransactions { - recipient: make_wallet("some_recipient"), + recipient: recipient.clone(), response_skeleton_opt: None, })) - .stop_the_system_after_last_msg(); + .start_scan_result(Ok(RetrieveTransactions { + recipient: recipient.clone(), + response_skeleton_opt: None, + })) + .finish_scan_result(None); let earning_wallet = make_wallet("earning"); let mut config = bc_from_earning_wallet(earning_wallet.clone()); config.scan_intervals_opt = Some(ScanIntervals { payable_scan_interval: Duration::from_secs(100), pending_payable_scan_interval: Duration::from_secs(10), - receivable_scan_interval: Duration::from_millis(99), + receivable_scan_interval: Duration::from_millis(15), }); let mut subject = AccountantBuilder::default() .bootstrapper_config(config) @@ -3663,8 +3711,25 @@ mod tests { ); let subject_addr = subject.start(); let subject_subs = Accountant::make_subs_from(&subject_addr); - let peer_actors = peer_actors_builder().build(); + let (peer_actors, peer_actors_addrs) = peer_actors_builder() + .blockchain_bridge(blockchain_bridge) + .build_with_addresses(); send_bind_message!(subject_subs, peer_actors); + let counter_msg = ReceivedPayments { + timestamp: SystemTime::now(), + new_start_block: BlockMarker::Value(1234), + transactions: vec![], + response_skeleton_opt: None, + }; + let counter_msg_setup = setup_for_counter_msg_triggered_via_type_id!( + RetrieveTransactions, + counter_msg, + subject_addr + ); + peer_actors_addrs + .blockchain_bridge_addr + .try_send(SetUpCounterMsgs::new(vec![counter_msg_setup])) + .unwrap(); subject_addr .try_send(ScanForReceivables { @@ -3702,37 +3767,37 @@ mod tests { assert!(start_scan_params.is_empty()); debug!( first_attempt_logger, - "first attempt verifying receivable scanner" + "first attempt receivable scanner logger verification" ); debug!( second_attempt_logger, - "second attempt verifying receivable scanner" + "second attempt receivable scanner logger verification" ); + let blockchain_bridge_recording = blockchain_bridge_recording_arc.lock().unwrap(); + let first_msg_towards_bb = + blockchain_bridge_recording.get_record::(0); + let expected_msg = RetrieveTransactions { + recipient, + response_skeleton_opt: None, + }; + assert_eq!(first_msg_towards_bb, &expected_msg); + let second_msg_towards_bb = + blockchain_bridge_recording.get_record::(1); + assert_eq!(second_msg_towards_bb, &expected_msg); assert_eq!( *notify_later_receivable_params, - vec![ - ( - ScanForReceivables { - response_skeleton_opt: None - }, - Duration::from_millis(99) - ), - ( - ScanForReceivables { - response_skeleton_opt: None - }, - Duration::from_millis(99) - ), - ] + vec![( + ScanForReceivables { + response_skeleton_opt: None + }, + Duration::from_millis(15) + ),] ); tlh.exists_log_containing(&format!( - "DEBUG: {test_name}: There was nothing to process during Receivables scan." - )); - tlh.exists_log_containing(&format!( - "DEBUG: {test_name}: first attempt verifying receivable scanner", + "DEBUG: {test_name}: first attempt receivable scanner logger verification", )); tlh.exists_log_containing(&format!( - "DEBUG: {test_name}: second attempt verifying receivable scanner", + "DEBUG: {test_name}: second attempt receivable scanner logger verification", )); } @@ -4207,7 +4272,8 @@ mod tests { expected = "internal error: entered unreachable code: Early stopped new payable scan \ was suggested to be followed up by the scan for Receivables, which is not supported though" )] - fn start_scan_error_in_new_payables_and_unexpected_reaction_by_receivable_scan_scheduling() { + fn start_scan_error_in_new_payables_and_unexpected_instruction_from_early_stop_scan_rescheduling( + ) { let mut subject = AccountantBuilder::default().build(); let reschedule_on_error_resolver = RescheduleScanOnErrorResolverMock::default() .resolve_rescheduling_on_error_result(ScanReschedulingAfterEarlyStop::Schedule( @@ -4427,32 +4493,6 @@ mod tests { ); } - #[test] - #[should_panic( - expected = "internal error: entered unreachable code: Early stopped pending payable scan \ - was suggested to be followed up by the scan for Receivables, which is not supported though" - )] - fn start_scan_error_in_pending_payables_and_unexpected_reaction_by_receivable_scan_scheduling() - { - let mut subject = AccountantBuilder::default().build(); - let reschedule_on_error_resolver = RescheduleScanOnErrorResolverMock::default() - .resolve_rescheduling_on_error_result(ScanReschedulingAfterEarlyStop::Schedule( - ScanType::Receivables, - )); - subject.scan_schedulers.reschedule_on_error_resolver = - Box::new(reschedule_on_error_resolver); - let system = System::new("test"); - let subject_addr = subject.start(); - - subject_addr - .try_send(ScanForPendingPayables { - response_skeleton_opt: None, - }) - .unwrap(); - - system.run(); - } - #[test] fn report_routing_service_provided_message_is_received() { init_test_logging(); @@ -4494,6 +4534,10 @@ mod tests { more_money_receivable_parameters[0], (now, make_wallet("booga"), (1 * 42) + (1234 * 24)) ); + TestLogHandler::new().exists_log_containing(&format!( + "TRACE: Accountant: Charging 29658 wei for routing of 1234 bytes to wallet {}", + paying_wallet + )); } #[test] @@ -4627,6 +4671,10 @@ mod tests { more_money_receivable_parameters[0], (now, make_wallet("booga"), (1 * 42) + (1234 * 24)) ); + TestLogHandler::new().exists_log_containing(&format!( + "TRACE: Accountant: Charging 29658 wei for exit service for 1234 bytes to wallet {}", + paying_wallet + )); } #[test] @@ -4793,6 +4841,19 @@ mod tests { ) ] ); + let test_log_handler = TestLogHandler::new(); + test_log_handler.exists_log_containing(&format!( + "TRACE: Accountant: MsgId 123: Accruing 36120 wei of debt to {} for consuming 1200 exited bytes", + earning_wallet_exit + )); + test_log_handler.exists_log_containing(&format!( + "TRACE: Accountant: MsgId 123: Accruing 82986 wei of debt to {} for consuming 3456 routed bytes", + earning_wallet_routing_1 + )); + test_log_handler.exists_log_containing(&format!( + "TRACE: Accountant: MsgId 123: Accruing 114100 wei of debt to {} for consuming 3456 routed bytes", + earning_wallet_routing_2 + )); } fn assert_that_we_do_not_charge_our_own_wallet_for_consumed_services( @@ -4984,7 +5045,14 @@ mod tests { .receivable_daos(vec![ForAccountantBody(receivable_dao)]) .build(); - let _ = subject.record_service_provided(i64::MAX as u64, 1, SystemTime::now(), 2, &wallet); + let _ = subject.record_service_provided( + i64::MAX as u64, + 1, + 1000, + SystemTime::now(), + 2, + &wallet, + ); } #[test] @@ -4997,7 +5065,7 @@ mod tests { .receivable_daos(vec![ForAccountantBody(receivable_dao)]) .build(); - subject.record_service_provided(i64::MAX as u64, 1, SystemTime::now(), 2, &wallet); + subject.record_service_provided(i64::MAX as u64, 1, 1000, SystemTime::now(), 2, &wallet); TestLogHandler::new().exists_log_containing(&format!( "ERROR: Accountant: Overflow error recording service provided for {}: service rate {}, byte rate 1, payload size 2. Skipping", @@ -5017,7 +5085,14 @@ mod tests { .build(); let service_rate = i64::MAX as u64; - subject.record_service_consumed(service_rate, 1, SystemTime::now(), 2, &wallet); + subject.record_service_consumed( + service_rate, + 1, + 9223372036854775809, + SystemTime::now(), + 2, + &wallet, + ); TestLogHandler::new().exists_log_containing(&format!( "ERROR: Accountant: Overflow error recording consumed services from {}: total charge {}, service rate {}, byte rate 1, payload size 2. Skipping", @@ -5042,7 +5117,14 @@ mod tests { .payable_daos(vec![ForAccountantBody(payable_dao)]) .build(); - let _ = subject.record_service_consumed(i64::MAX as u64, 1, SystemTime::now(), 2, &wallet); + let _ = subject.record_service_consumed( + i64::MAX as u64, + 1, + 1000, + SystemTime::now(), + 2, + &wallet, + ); } #[test] @@ -5090,8 +5172,8 @@ mod tests { Box::new(NotifyHandleMock::default().panic_on_schedule_attempt()); subject.scan_schedulers.payable.new_payable_notify_later = Box::new(NotifyLaterHandleMock::default().panic_on_schedule_attempt()); - subject.scan_schedulers.payable.retry_payable_notify = - Box::new(NotifyHandleMock::default().panic_on_schedule_attempt()); + subject.scan_schedulers.payable.retry_payable_notify_later = + Box::new(NotifyLaterHandleMock::default().panic_on_schedule_attempt()); let expected_tx = TxBuilder::default().hash(expected_hash.clone()).build(); let sent_payable = SentPayables { payment_procedure_result: Ok(BatchResults { @@ -5149,8 +5231,8 @@ mod tests { Box::new(NotifyHandleMock::default().panic_on_schedule_attempt()); subject.scan_schedulers.payable.new_payable_notify_later = Box::new(NotifyLaterHandleMock::default().panic_on_schedule_attempt()); - subject.scan_schedulers.payable.retry_payable_notify = - Box::new(NotifyHandleMock::default().panic_on_schedule_attempt()); + subject.scan_schedulers.payable.retry_payable_notify_later = + Box::new(NotifyLaterHandleMock::default().panic_on_schedule_attempt()); let expected_tx = TxBuilder::default().hash(expected_hash.clone()).build(); let sent_payable = SentPayables { payment_procedure_result: Ok(BatchResults { @@ -5184,7 +5266,7 @@ mod tests { init_test_logging(); let test_name = "retry_payable_scan_is_requested_to_be_repeated"; let finish_scan_params_arc = Arc::new(Mutex::new(vec![])); - let retry_payable_notify_params_arc = Arc::new(Mutex::new(vec![])); + let retry_payable_notify_later_params_arc = Arc::new(Mutex::new(vec![])); let system = System::new(test_name); let consuming_wallet = make_paying_wallet(b"paying wallet"); let mut subject = AccountantBuilder::default() @@ -5201,8 +5283,10 @@ mod tests { result: NextScanToRun::RetryPayableScan, }), ))); - subject.scan_schedulers.payable.retry_payable_notify = - Box::new(NotifyHandleMock::default().notify_params(&retry_payable_notify_params_arc)); + subject.scan_schedulers.payable.retry_payable_notify_later = Box::new( + NotifyLaterHandleMock::default() + .notify_later_params(&retry_payable_notify_later_params_arc), + ); subject.scan_schedulers.payable.new_payable_notify = Box::new(NotifyHandleMock::default().panic_on_schedule_attempt()); subject.scan_schedulers.payable.new_payable_notify_later = @@ -5228,9 +5312,10 @@ mod tests { let (actual_sent_payable, logger) = finish_scan_params.remove(0); assert_eq!(actual_sent_payable, sent_payable,); assert_using_the_same_logger(&logger, test_name, None); - let mut payable_notify_params = retry_payable_notify_params_arc.lock().unwrap(); - let scheduled_msg = payable_notify_params.remove(0); + let mut payable_notify_params = retry_payable_notify_later_params_arc.lock().unwrap(); + let (scheduled_msg, duration) = payable_notify_params.remove(0); assert_eq!(scheduled_msg, ScanForRetryPayables::default()); + assert_eq!(duration, Duration::from_secs(5 * 60)); assert!( payable_notify_params.is_empty(), "Should be empty but {:?}", @@ -5245,7 +5330,7 @@ mod tests { let test_name = "accountant_in_automatic_mode_schedules_tx_retry_as_some_pending_payables_have_not_completed"; let finish_scan_params_arc = Arc::new(Mutex::new(vec![])); - let retry_payable_notify_params_arc = Arc::new(Mutex::new(vec![])); + let retry_payable_notify_later_params_arc = Arc::new(Mutex::new(vec![])); let mut subject = AccountantBuilder::default() .logger(Logger::new(test_name)) .build(); @@ -5263,8 +5348,10 @@ mod tests { Box::new(NotifyLaterHandleMock::default().panic_on_schedule_attempt()); subject.scan_schedulers.pending_payable.handle = Box::new(NotifyLaterHandleMock::default().panic_on_schedule_attempt()); - subject.scan_schedulers.payable.retry_payable_notify = - Box::new(NotifyHandleMock::default().notify_params(&retry_payable_notify_params_arc)); + subject.scan_schedulers.payable.retry_payable_notify_later = Box::new( + NotifyLaterHandleMock::default() + .notify_later_params(&retry_payable_notify_later_params_arc), + ); let system = System::new(test_name); let (mut msg, _) = make_tx_receipts_msg(vec![ SeedsToMakeUpPayableWithStatus { @@ -5286,12 +5373,15 @@ mod tests { let mut finish_scan_params = finish_scan_params_arc.lock().unwrap(); let (msg_actual, logger) = finish_scan_params.remove(0); assert_eq!(msg_actual, msg); - let retry_payable_notify_params = retry_payable_notify_params_arc.lock().unwrap(); + let retry_payable_notify_params = retry_payable_notify_later_params_arc.lock().unwrap(); assert_eq!( *retry_payable_notify_params, - vec![ScanForRetryPayables { - response_skeleton_opt: None - }] + vec![( + ScanForRetryPayables { + response_skeleton_opt: None + }, + Duration::from_secs(5 * 60) + )] ); assert_using_the_same_logger(&logger, test_name, None) } @@ -5314,8 +5404,8 @@ mod tests { .replace_scanner(ScannerReplacement::PendingPayable(ReplacementType::Mock( pending_payable_scanner, ))); - subject.scan_schedulers.payable.retry_payable_notify = - Box::new(NotifyHandleMock::default().panic_on_schedule_attempt()); + subject.scan_schedulers.payable.retry_payable_notify_later = + Box::new(NotifyLaterHandleMock::default().panic_on_schedule_attempt()); subject.scan_schedulers.payable.new_payable_notify = Box::new(NotifyHandleMock::default().panic_on_schedule_attempt()); subject.scan_schedulers.payable.new_payable_notify_later = @@ -5380,8 +5470,8 @@ mod tests { .replace_scanner(ScannerReplacement::PendingPayable(ReplacementType::Mock( pending_payable_scanner, ))); - subject.scan_schedulers.payable.retry_payable_notify = - Box::new(NotifyHandleMock::default().panic_on_schedule_attempt()); + subject.scan_schedulers.payable.retry_payable_notify_later = + Box::new(NotifyLaterHandleMock::default().panic_on_schedule_attempt()); subject.scan_schedulers.payable.new_payable_notify = Box::new(NotifyHandleMock::default().panic_on_schedule_attempt()); subject.scan_schedulers.payable.new_payable_notify_later = @@ -5443,8 +5533,9 @@ mod tests { .replace_scanner(ScannerReplacement::PendingPayable(ReplacementType::Mock( pending_payable_scanner, ))); - subject.scan_schedulers.payable.retry_payable_notify = - Box::new(NotifyHandleMock::default().notify_params(&retry_payable_notify_params_arc)); + subject.scan_schedulers.payable.retry_payable_notify_later = Box::new( + NotifyLaterHandleMock::default().notify_later_params(&retry_payable_notify_params_arc), + ); subject.scan_schedulers.payable.new_payable_notify = Box::new(NotifyHandleMock::default().panic_on_schedule_attempt()); subject.scan_schedulers.payable.new_payable_notify_later = @@ -5468,9 +5559,12 @@ mod tests { let retry_payable_notify_params = retry_payable_notify_params_arc.lock().unwrap(); assert_eq!( *retry_payable_notify_params, - vec![ScanForRetryPayables { - response_skeleton_opt: Some(response_skeleton) - }] + vec![( + ScanForRetryPayables { + response_skeleton_opt: Some(response_skeleton) + }, + Duration::from_secs(5 * 60) + )] ); assert_using_the_same_logger(&logger, test_name, None) } @@ -5481,7 +5575,7 @@ mod tests { let test_name = "accountant_confirms_all_pending_txs_and_schedules_new_payable_scanner_timely"; let finish_scan_params_arc = Arc::new(Mutex::new(vec![])); - let time_until_next_scan_params_arc = Arc::new(Mutex::new(vec![])); + let compute_time_to_next_scan_params_arc = Arc::new(Mutex::new(vec![])); let new_payable_notify_later_arc = Arc::new(Mutex::new(vec![])); let new_payable_notify_arc = Arc::new(Mutex::new(vec![])); let system = System::new("new_payable_scanner_timely"); @@ -5498,9 +5592,9 @@ mod tests { ))); let expected_computed_interval = Duration::from_secs(3); let interval_computer = NewPayableScanIntervalComputerMock::default() - .time_until_next_scan_params(&time_until_next_scan_params_arc) + .compute_time_to_next_scan_params(&compute_time_to_next_scan_params_arc) // This determines the test - .time_until_next_scan_result(ScanTiming::WaitFor(expected_computed_interval)); + .compute_time_to_next_scan_result(ScanTiming::WaitFor(expected_computed_interval)); subject.scan_schedulers.payable.interval_computer = Box::new(interval_computer); subject.scan_schedulers.payable.new_payable_notify_later = Box::new( NotifyLaterHandleMock::default().notify_later_params(&new_payable_notify_later_arc), @@ -5558,7 +5652,7 @@ mod tests { let test_name = "accountant_confirms_payable_txs_and_schedules_the_delayed_new_payable_scanner_asap"; let finish_scan_params_arc = Arc::new(Mutex::new(vec![])); - let time_until_next_scan_params_arc = Arc::new(Mutex::new(vec![])); + let compute_time_to_next_scan_params_arc = Arc::new(Mutex::new(vec![])); let new_payable_notify_later_arc = Arc::new(Mutex::new(vec![])); let new_payable_notify_arc = Arc::new(Mutex::new(vec![])); let mut subject = AccountantBuilder::default() @@ -5573,9 +5667,9 @@ mod tests { pending_payable_scanner, ))); let interval_computer = NewPayableScanIntervalComputerMock::default() - .time_until_next_scan_params(&time_until_next_scan_params_arc) + .compute_time_to_next_scan_params(&compute_time_to_next_scan_params_arc) // This determines the test - .time_until_next_scan_result(ScanTiming::ReadyNow); + .compute_time_to_next_scan_result(ScanTiming::ReadyNow); subject.scan_schedulers.payable.interval_computer = Box::new(interval_computer); subject.scan_schedulers.payable.new_payable_notify_later = Box::new( NotifyLaterHandleMock::default().notify_later_params(&new_payable_notify_later_arc), @@ -5610,8 +5704,8 @@ mod tests { "Should be empty but {:?}", finish_scan_params ); - let time_until_next_scan_params = time_until_next_scan_params_arc.lock().unwrap(); - assert_eq!(*time_until_next_scan_params, vec![()]); + let compute_time_to_next_scan_params = compute_time_to_next_scan_params_arc.lock().unwrap(); + assert_eq!(*compute_time_to_next_scan_params, vec![()]); let new_payable_notify_later = new_payable_notify_later_arc.lock().unwrap(); assert!( new_payable_notify_later.is_empty(), @@ -5662,7 +5756,7 @@ mod tests { }), }]); let left_side_bound = if let ScanTiming::WaitFor(interval) = - assertion_interval_computer.time_until_next_scan() + assertion_interval_computer.compute_time_to_next_scan() { interval } else { @@ -5676,7 +5770,7 @@ mod tests { let new_payable_notify_later = new_payable_notify_later_arc.lock().unwrap(); let (_, actual_interval) = new_payable_notify_later[0]; let right_side_bound = if let ScanTiming::WaitFor(interval) = - assertion_interval_computer.time_until_next_scan() + assertion_interval_computer.compute_time_to_next_scan() { interval } else { @@ -5854,8 +5948,9 @@ mod tests { // Setup let notify_later_params_arc = Arc::new(Mutex::new(vec![])); scan_schedulers.payable.interval_computer = Box::new( - NewPayableScanIntervalComputerMock::default() - .time_until_next_scan_result(ScanTiming::WaitFor(Duration::from_secs(152))), + NewPayableScanIntervalComputerMock::default().compute_time_to_next_scan_result( + ScanTiming::WaitFor(Duration::from_secs(152)), + ), ); scan_schedulers.payable.new_payable_notify_later = Box::new( NotifyLaterHandleMock::default().notify_later_params(¬ify_later_params_arc), @@ -5887,28 +5982,32 @@ mod tests { Box::new( |_scanners: &mut Scanners, scan_schedulers: &mut ScanSchedulers| { // Setup - let notify_params_arc = Arc::new(Mutex::new(vec![])); - scan_schedulers.payable.retry_payable_notify = - Box::new(NotifyHandleMock::default().notify_params(¬ify_params_arc)); + let notify_later_params_arc = Arc::new(Mutex::new(vec![])); + scan_schedulers.payable.retry_payable_notify_later = Box::new( + NotifyLaterHandleMock::default().notify_later_params(¬ify_later_params_arc), + ); // Assertions Box::new(move |response_skeleton_opt| { - let notify_params = notify_params_arc.lock().unwrap(); + let notify_later_params = notify_later_params_arc.lock().unwrap(); match response_skeleton_opt { None => { // Response skeleton must be None assert_eq!( - *notify_params, - vec![ScanForRetryPayables { - response_skeleton_opt: None - }] + *notify_later_params, + vec![( + ScanForRetryPayables { + response_skeleton_opt: None + }, + Duration::from_secs(5 * 60) + )] ) } Some(_) => { assert!( - notify_params.is_empty(), + notify_later_params.is_empty(), "Should be empty but contained {:?}", - notify_params + notify_later_params ) } } diff --git a/node/src/accountant/scanners/mod.rs b/node/src/accountant/scanners/mod.rs index 787b9a8b2..d14814a5b 100644 --- a/node/src/accountant/scanners/mod.rs +++ b/node/src/accountant/scanners/mod.rs @@ -122,7 +122,10 @@ impl Scanners { }); } - Self::start_correct_payable_scanner::( + <(dyn MultistageDualPayableScanner) as StartableScanner< + ScanForNewPayables, + InitialTemplatesMessage, + >>::start_scan( &mut *self.payable, wallet, timestamp, @@ -150,7 +153,10 @@ impl Scanners { ) } - Self::start_correct_payable_scanner::( + <(dyn MultistageDualPayableScanner) as StartableScanner< + ScanForRetryPayables, + InitialTemplatesMessage, + >>::start_scan( &mut *self.payable, wallet, timestamp, @@ -168,10 +174,14 @@ impl Scanners { automatic_scans_enabled: bool, ) -> Result { let triggered_manually = response_skeleton_opt.is_some(); - self.check_general_conditions_for_pending_payable_scan( - triggered_manually, - automatic_scans_enabled, - )?; + if triggered_manually && automatic_scans_enabled { + return Err(StartScanError::ManualTriggerError( + ManulTriggerError::AutomaticScanConflict, + )); + } + + self.check_pending_payable_existence(triggered_manually)?; + match ( self.pending_payable.scan_started_at(), self.payable.scan_started_at(), @@ -256,6 +266,7 @@ impl Scanners { } pub fn acknowledge_scan_error(&mut self, error: &ScanError, logger: &Logger) { + debug!(logger, "Acknowledging a scan that couldn't finish"); match error.scan_type { DetailedScanType::NewPayables | DetailedScanType::RetryPayables => { self.payable.mark_as_ended(logger) @@ -298,41 +309,14 @@ impl Scanners { self.initial_pending_payable_scan = false } - // This is a helper function reducing a boilerplate of complex trait resolving where - // the compiler requires to specify which trigger message distinguishes the scan to run. - // The payable scanner offers two modes through doubled implementations of StartableScanner - // which uses the trigger message type as the only distinction between them. - fn start_correct_payable_scanner<'a, TriggerMessage>( - scanner: &'a mut (dyn MultistageDualPayableScanner + 'a), - wallet: &Wallet, - timestamp: SystemTime, - response_skeleton_opt: Option, - logger: &Logger, - ) -> Result - where - TriggerMessage: Message, - (dyn MultistageDualPayableScanner + 'a): - StartableScanner, - { - <(dyn MultistageDualPayableScanner + 'a) as StartableScanner< - TriggerMessage, - InitialTemplatesMessage, - >>::start_scan(scanner, wallet, timestamp, response_skeleton_opt, logger) - } - - fn check_general_conditions_for_pending_payable_scan( + fn check_pending_payable_existence( &mut self, triggered_manually: bool, - automatic_scans_enabled: bool, ) -> Result<(), StartScanError> { - if triggered_manually && automatic_scans_enabled { - return Err(StartScanError::ManualTriggerError( - ManulTriggerError::AutomaticScanConflict, - )); - } if self.initial_pending_payable_scan { return Ok(()); } + if triggered_manually && !self.aware_of_unresolved_pending_payable { return Err(StartScanError::ManualTriggerError( ManulTriggerError::UnnecessaryRequest { @@ -794,11 +778,11 @@ mod tests { false ); let dumped_records = pending_payable_scanner - .yet_unproven_failed_payables + .suspected_failed_payables .dump_cache(); assert!( dumped_records.is_empty(), - "There should be no yet unproven failures but found {:?}.", + "There should be no suspected failures but found {:?}.", dumped_records ); assert_eq!( @@ -1199,7 +1183,9 @@ mod tests { ); TestLogHandler::new().assert_logs_match_in_order(vec![ &format!("INFO: {test_name}: Scanning for pending payable"), - &format!("DEBUG: {test_name}: Found 1 pending payables and 1 unfinalized failures to process"), + &format!( + "DEBUG: {test_name}: Found 1 pending payables and 1 suspected failures to process" + ), ]) } @@ -1340,8 +1326,8 @@ mod tests { #[test] #[should_panic( - expected = "internal error: entered unreachable code: Automatic pending payable \ - scan should never start if there are no pending payables to process." + expected = "internal error: entered unreachable code: Automatic pending payable scan should \ + never start if there are no pending payables to process." )] fn pending_payable_scanner_bumps_into_zero_pending_payable_awareness_in_the_automatic_mode() { let consuming_wallet = make_paying_wallet(b"consuming"); @@ -1360,11 +1346,12 @@ mod tests { } #[test] - fn check_general_conditions_for_pending_payable_scan_if_it_is_initial_pending_payable_scan() { + fn check_pending_payable_existence_for_initial_pending_payable_scan_and_zero_awareness() { let mut subject = make_dull_subject(); + subject.aware_of_unresolved_pending_payable = false; subject.initial_pending_payable_scan = true; - let result = subject.check_general_conditions_for_pending_payable_scan(false, true); + let result = subject.check_pending_payable_existence(false); assert_eq!(result, Ok(())); assert_eq!(subject.initial_pending_payable_scan, true); diff --git a/node/src/accountant/scanners/payable_scanner/mod.rs b/node/src/accountant/scanners/payable_scanner/mod.rs index b82d2c46e..4c9ac9804 100644 --- a/node/src/accountant/scanners/payable_scanner/mod.rs +++ b/node/src/accountant/scanners/payable_scanner/mod.rs @@ -251,6 +251,10 @@ impl PayableScanner { self.insert_records_in_sent_payables(&batch_results.sent_txs); } if failed > 0 { + debug!( + logger, + "Recording failed txs: {:?}", batch_results.failed_txs + ); self.insert_records_in_failed_payables(&batch_results.failed_txs); } } diff --git a/node/src/accountant/scanners/payable_scanner/start_scan.rs b/node/src/accountant/scanners/payable_scanner/start_scan.rs index 35cbd3ab2..2684407f7 100644 --- a/node/src/accountant/scanners/payable_scanner/start_scan.rs +++ b/node/src/accountant/scanners/payable_scanner/start_scan.rs @@ -66,7 +66,7 @@ impl StartableScanner for Payable info!(logger, "Scanning for retry payables"); let failed_txs = self.get_txs_to_retry(); let amount_from_payables = self.find_amount_from_payables(&failed_txs); - let retry_tx_templates = RetryTxTemplates::new(&failed_txs, &amount_from_payables); + let retry_tx_templates = RetryTxTemplates::new(&failed_txs, &amount_from_payables, logger); Ok(InitialTemplatesMessage { initial_templates: Either::Right(retry_tx_templates), @@ -89,7 +89,7 @@ mod tests { use crate::accountant::scanners::payable_scanner::tx_templates::initial::retry::{ RetryTxTemplate, RetryTxTemplates, }; - use crate::accountant::scanners::Scanners; + use crate::accountant::scanners::payable_scanner::MultistageDualPayableScanner; use crate::accountant::test_utils::{ make_payable_account, FailedPayableDaoMock, PayableDaoMock, }; @@ -144,7 +144,10 @@ mod tests { .payable_dao(payable_dao) .build(); - let result = Scanners::start_correct_payable_scanner::( + let result = <(dyn MultistageDualPayableScanner) as StartableScanner< + ScanForRetryPayables, + InitialTemplatesMessage, + >>::start_scan( &mut subject, &consuming_wallet, timestamp, @@ -157,11 +160,8 @@ mod tests { let retrieve_payables_params = retrieve_payables_params_arc.lock().unwrap(); let expected_tx_templates = { let mut tx_template_1 = RetryTxTemplate::from(&failed_tx_1); - tx_template_1.base.amount_in_wei = - tx_template_1.base.amount_in_wei + payable_account_1.balance_wei; - + tx_template_1.base.amount_in_wei = payable_account_1.balance_wei; let tx_template_2 = RetryTxTemplate::from(&failed_tx_2); - RetryTxTemplates(vec![tx_template_1, tx_template_2]) }; assert_eq!( diff --git a/node/src/accountant/scanners/payable_scanner/tx_templates/initial/retry.rs b/node/src/accountant/scanners/payable_scanner/tx_templates/initial/retry.rs index 9990635cd..8157a373b 100644 --- a/node/src/accountant/scanners/payable_scanner/tx_templates/initial/retry.rs +++ b/node/src/accountant/scanners/payable_scanner/tx_templates/initial/retry.rs @@ -1,6 +1,7 @@ // Copyright (c) 2025, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. use crate::accountant::db_access_objects::failed_payable_dao::FailedTx; use crate::accountant::scanners::payable_scanner::tx_templates::BaseTxTemplate; +use masq_lib::logger::Logger; use std::collections::{BTreeSet, HashMap}; use std::ops::{Deref, DerefMut}; use web3::types::Address; @@ -13,11 +14,25 @@ pub struct RetryTxTemplate { } impl RetryTxTemplate { - pub fn new(failed_tx: &FailedTx, payable_scan_amount_opt: Option) -> Self { + pub fn new( + failed_tx: &FailedTx, + updated_payable_balance_opt: Option, + logger: &Logger, + ) -> Self { let mut retry_template = RetryTxTemplate::from(failed_tx); - if let Some(payable_scan_amount) = payable_scan_amount_opt { - retry_template.base.amount_in_wei += payable_scan_amount; + debug!(logger, "Tx to retry {:?}", failed_tx); + + if let Some(updated_payable_balance) = updated_payable_balance_opt { + debug!( + logger, + "Updating the pay for {:?} from former {} to latest accounted balance {} of minor", + failed_tx.receiver_address, + failed_tx.amount_minor, + updated_payable_balance + ); + + retry_template.base.amount_in_wei = updated_payable_balance; } retry_template @@ -44,6 +59,7 @@ impl RetryTxTemplates { pub fn new( txs_to_retry: &BTreeSet, amounts_from_payables: &HashMap, + logger: &Logger, ) -> Self { Self( txs_to_retry @@ -52,7 +68,7 @@ impl RetryTxTemplates { let payable_scan_amount_opt = amounts_from_payables .get(&tx_to_retry.receiver_address) .copied(); - RetryTxTemplate::new(tx_to_retry, payable_scan_amount_opt) + RetryTxTemplate::new(tx_to_retry, payable_scan_amount_opt, logger) }) .collect(), ) @@ -98,6 +114,49 @@ mod tests { }; use crate::accountant::scanners::payable_scanner::tx_templates::BaseTxTemplate; use crate::blockchain::test_utils::{make_address, make_tx_hash}; + use masq_lib::logger::Logger; + + #[test] + fn retry_tx_template_constructor_works() { + let receiver_address = make_address(42); + let amount_in_wei = 1_000_000; + let gas_price = 20_000_000_000; + let nonce = 123; + let tx_hash = make_tx_hash(789); + let failed_tx = FailedTx { + hash: tx_hash, + receiver_address, + amount_minor: amount_in_wei, + gas_price_minor: gas_price, + nonce, + timestamp: 1234567, + reason: FailureReason::PendingTooLong, + status: FailureStatus::RetryRequired, + }; + let logger = Logger::new("test"); + let fetched_balance_from_payable_table_opt_1 = None; + let fetched_balance_from_payable_table_opt_2 = Some(1_234_567); + + let result_1 = RetryTxTemplate::new( + &failed_tx, + fetched_balance_from_payable_table_opt_1, + &logger, + ); + let result_2 = RetryTxTemplate::new( + &failed_tx, + fetched_balance_from_payable_table_opt_2, + &logger, + ); + + let assert = |result: RetryTxTemplate, expected_amount_in_wei: u128| { + assert_eq!(result.base.receiver_address, receiver_address); + assert_eq!(result.base.amount_in_wei, expected_amount_in_wei); + assert_eq!(result.prev_gas_price_wei, gas_price); + assert_eq!(result.prev_nonce, nonce); + }; + assert(result_1, amount_in_wei); + assert(result_2, fetched_balance_from_payable_table_opt_2.unwrap()); + } #[test] fn retry_tx_template_can_be_created_from_failed_tx() { diff --git a/node/src/accountant/scanners/payable_scanner/tx_templates/priced/new.rs b/node/src/accountant/scanners/payable_scanner/tx_templates/priced/new.rs index 6de54e4c9..200b9602b 100644 --- a/node/src/accountant/scanners/payable_scanner/tx_templates/priced/new.rs +++ b/node/src/accountant/scanners/payable_scanner/tx_templates/priced/new.rs @@ -4,7 +4,7 @@ use crate::accountant::scanners::payable_scanner::tx_templates::initial::new::{ NewTxTemplate, NewTxTemplates, }; use crate::accountant::scanners::payable_scanner::tx_templates::BaseTxTemplate; -use crate::blockchain::blockchain_bridge::increase_gas_price_by_margin; +use crate::blockchain::blockchain_bridge::increase_by_percentage; use masq_lib::logger::Logger; use std::ops::Deref; use thousands::Separable; @@ -63,7 +63,7 @@ impl PricedNewTxTemplates { ceil: u128, logger: &Logger, ) -> Self { - let computed_gas_price_wei = increase_gas_price_by_margin(latest_gas_price_wei); + let computed_gas_price_wei = increase_by_percentage(latest_gas_price_wei); let safe_gas_price_wei = if computed_gas_price_wei > ceil { warning!( diff --git a/node/src/accountant/scanners/payable_scanner/tx_templates/priced/retry.rs b/node/src/accountant/scanners/payable_scanner/tx_templates/priced/retry.rs index 48e41f4b9..3477b206a 100644 --- a/node/src/accountant/scanners/payable_scanner/tx_templates/priced/retry.rs +++ b/node/src/accountant/scanners/payable_scanner/tx_templates/priced/retry.rs @@ -4,7 +4,8 @@ use crate::accountant::scanners::payable_scanner::tx_templates::initial::retry:: RetryTxTemplate, RetryTxTemplates, }; use crate::accountant::scanners::payable_scanner::tx_templates::BaseTxTemplate; -use crate::blockchain::blockchain_bridge::increase_gas_price_by_margin; +use crate::blockchain::blockchain_bridge::increase_by_percentage; +use masq_lib::constants::DEFAULT_GAS_PRICE_RETRY_CONSTANT; use masq_lib::logger::Logger; use std::ops::{Deref, DerefMut}; use thousands::Separable; @@ -34,7 +35,7 @@ impl PricedRetryTxTemplate { ) -> PricedRetryTxTemplate { let receiver = retry_tx_template.base.receiver_address; let computed_gas_price_wei = - Self::compute_gas_price(retry_tx_template.prev_gas_price_wei, latest_gas_price_wei); + Self::compute_gas_price(latest_gas_price_wei, retry_tx_template.prev_gas_price_wei); let safe_gas_price_wei = if computed_gas_price_wei > ceil { log_builder.push(receiver, computed_gas_price_wei); @@ -46,10 +47,12 @@ impl PricedRetryTxTemplate { PricedRetryTxTemplate::new(retry_tx_template, safe_gas_price_wei) } - fn compute_gas_price(latest_gas_price_wei: u128, prev_gas_price_wei: u128) -> u128 { - let gas_price_wei = latest_gas_price_wei.max(prev_gas_price_wei); - - increase_gas_price_by_margin(gas_price_wei) + pub fn compute_gas_price(latest_gas_price_wei: u128, prev_gas_price_wei: u128) -> u128 { + if latest_gas_price_wei >= prev_gas_price_wei { + increase_by_percentage(latest_gas_price_wei) + } else { + prev_gas_price_wei + DEFAULT_GAS_PRICE_RETRY_CONSTANT + } } } @@ -167,3 +170,44 @@ impl RetryLogBuilder { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compute_gas_price_increases_by_percentage_if_latest_is_higher() { + let latest_gas_price_wei = 101; + let prev_gas_price_wei = 100; + + let computed_gas_price = + PricedRetryTxTemplate::compute_gas_price(latest_gas_price_wei, prev_gas_price_wei); + + let expected_gas_price = increase_by_percentage(latest_gas_price_wei); + assert_eq!(computed_gas_price, expected_gas_price); + } + + #[test] + fn compute_gas_price_increases_by_percentage_if_latest_is_equal() { + let latest_gas_price_wei = 100; + let prev_gas_price_wei = 100; + + let computed_gas_price = + PricedRetryTxTemplate::compute_gas_price(latest_gas_price_wei, prev_gas_price_wei); + + let expected_gas_price = increase_by_percentage(latest_gas_price_wei); + assert_eq!(computed_gas_price, expected_gas_price); + } + + #[test] + fn compute_gas_price_increments_previous_by_constant_if_latest_is_lower() { + let latest_gas_price_wei = 99; + let prev_gas_price_wei = 100; + + let computed_gas_price = + PricedRetryTxTemplate::compute_gas_price(latest_gas_price_wei, prev_gas_price_wei); + + let expected_gas_price = prev_gas_price_wei + DEFAULT_GAS_PRICE_RETRY_CONSTANT; + assert_eq!(computed_gas_price, expected_gas_price); + } +} diff --git a/node/src/accountant/scanners/pending_payable_scanner/mod.rs b/node/src/accountant/scanners/pending_payable_scanner/mod.rs index 499ee0179..4fe12add1 100644 --- a/node/src/accountant/scanners/pending_payable_scanner/mod.rs +++ b/node/src/accountant/scanners/pending_payable_scanner/mod.rs @@ -65,7 +65,7 @@ pub struct PendingPayableScanner { pub failed_payable_dao: Box, pub financial_statistics: Rc>, pub current_sent_payables: Box>, - pub yet_unproven_failed_payables: Box>, + pub suspected_failed_payables: Box>, pub clock: Box, } @@ -119,6 +119,8 @@ impl Scanner for PendingPayableScan let retry_opt = scan_report.requires_payments_retry(); + debug!(logger, "Payment retry requirement: {:?}", retry_opt); + self.process_txs_by_state(scan_report, logger); self.mark_as_ended(logger); @@ -136,7 +138,7 @@ impl Scanner for PendingPayableScan impl CachesEmptiableScanner for PendingPayableScanner { fn empty_caches(&mut self, logger: &Logger) { self.current_sent_payables.ensure_empty_cache(logger); - self.yet_unproven_failed_payables.ensure_empty_cache(logger); + self.suspected_failed_payables.ensure_empty_cache(logger); } } @@ -155,14 +157,16 @@ impl PendingPayableScanner { failed_payable_dao, financial_statistics, current_sent_payables: Box::new(CurrentPendingPayables::default()), - yet_unproven_failed_payables: Box::new(RecheckRequiringFailures::default()), + suspected_failed_payables: Box::new(RecheckRequiringFailures::default()), clock: Box::new(SimpleClockReal::default()), } } fn harvest_tables(&mut self, logger: &Logger) -> Result, StartScanError> { + debug!(logger, "Harvesting sent_payable and failed_payable tables"); + let pending_tx_hashes_opt = self.harvest_pending_payables(); - let failure_hashes_opt = self.harvest_unproven_failures(); + let failure_hashes_opt = self.harvest_suspected_failures(); if Self::is_there_nothing_to_process( pending_tx_hashes_opt.as_ref(), @@ -199,7 +203,7 @@ impl PendingPayableScanner { Some(pending_tx_hashes) } - fn harvest_unproven_failures(&mut self) -> Option> { + fn harvest_suspected_failures(&mut self) -> Option> { let failures = self .failed_payable_dao .retrieve_txs(Some(FailureRetrieveCondition::EveryRecheckRequiredRecord)) @@ -211,7 +215,7 @@ impl PendingPayableScanner { } let failure_hashes = Self::wrap_hashes(&failures, TxHashByTable::FailedPayable); - self.yet_unproven_failed_payables.load_cache(failures); + self.suspected_failed_payables.load_cache(failures); Some(failure_hashes) } @@ -250,8 +254,8 @@ impl PendingPayableScanner { fn emptiness_check(&self, msg: &TxReceiptsMessage) { if msg.results.is_empty() { panic!( - "We should never receive an empty list of results. \ - Even receipts that could not be retrieved can be interpreted" + "We should never receive an empty list of results. Even receipts that could \ + not be retrieved can be interpreted" ) } } @@ -325,7 +329,7 @@ impl PendingPayableScanner { }; self.current_sent_payables.ensure_empty_cache(logger); - self.yet_unproven_failed_payables.ensure_empty_cache(logger); + self.suspected_failed_payables.ensure_empty_cache(logger); cases } @@ -350,10 +354,7 @@ impl PendingPayableScanner { } } TxHashByTable::FailedPayable(tx_hash) => { - match self - .yet_unproven_failed_payables - .get_record_by_hash(tx_hash) - { + match self.suspected_failed_payables.get_record_by_hash(tx_hash) { Some(failed_tx) => { cases.push(TxCaseToBeInterpreted::new( TxByTable::FailedPayable(failed_tx), @@ -378,10 +379,10 @@ impl PendingPayableScanner { panic!( "Looking up '{:?}' in the cache, the record could not be found. Dumping \ - the remaining values. Pending payables: {:?}. Unproven failures: {:?}.", + the remaining values. Pending payables: {:?}. Suspected failures: {:?}.", missing_entry, rearrange(self.current_sent_payables.dump_cache()), - rearrange(self.yet_unproven_failed_payables.dump_cache()), + rearrange(self.suspected_failed_payables.dump_cache()), ) } @@ -396,14 +397,18 @@ impl PendingPayableScanner { logger: &Logger, ) { self.handle_tx_failure_reclaims(confirmed_txs.reclaims, logger); - self.handle_normal_confirmations(confirmed_txs.normal_confirmations, logger); + self.handle_standard_confirmations(confirmed_txs.standard_confirmations, logger); } fn handle_tx_failure_reclaims(&mut self, reclaimed: Vec, logger: &Logger) { if reclaimed.is_empty() { + debug!(logger, "No failure reclaim to process"); + return; } + debug!(logger, "Processing failure reclaims: {:?}", reclaimed); + let hashes_and_blocks = Self::collect_and_sort_hashes_and_blocks(&reclaimed); self.replace_sent_tx_records(&reclaimed, &hashes_and_blocks, logger); @@ -495,11 +500,19 @@ impl PendingPayableScanner { } } - fn handle_normal_confirmations(&mut self, confirmed_txs: Vec, logger: &Logger) { + fn handle_standard_confirmations(&mut self, confirmed_txs: Vec, logger: &Logger) { if confirmed_txs.is_empty() { + debug!(logger, "No standard tx confirmations to process"); return; } + debug!( + logger, + "Processing {} standard tx confirmations", + confirmed_txs.len() + ); + trace!(logger, "{:?}", confirmed_txs); + self.confirm_transactions(&confirmed_txs); self.update_tx_blocks(&confirmed_txs, logger); @@ -616,7 +629,7 @@ impl PendingPayableScanner { }); self.add_new_failures(grouped_failures.new_failures, logger); - self.finalize_unproven_failures(grouped_failures.rechecks_completed, logger); + self.finalize_suspected_failures(grouped_failures.rechecks_completed, logger); } fn add_new_failures(&self, new_failures: Vec, logger: &Logger) { @@ -632,9 +645,12 @@ impl PendingPayableScanner { } if new_failures.is_empty() { + debug!(logger, "No reverted txs to process"); return; } + debug!(logger, "Processing reverted txs {:?}", new_failures); + let new_failures_btree_set: BTreeSet = new_failures.iter().cloned().collect(); if let Err(e) = self @@ -665,7 +681,7 @@ impl PendingPayableScanner { } } - fn finalize_unproven_failures(&self, rechecks_completed: Vec, logger: &Logger) { + fn finalize_suspected_failures(&self, rechecks_completed: Vec, logger: &Logger) { fn prepare_hashmap(rechecks_completed: &[TxHash]) -> HashMap { rechecks_completed .iter() @@ -674,9 +690,17 @@ impl PendingPayableScanner { } if rechecks_completed.is_empty() { + debug!(logger, "No recheck-requiring failures to finalize"); return; } + debug!( + logger, + "Finalizing {} double-checked failures", + rechecks_completed.len() + ); + trace!(logger, "{:?}", rechecks_completed); + match self .failed_payable_dao .update_statuses(&prepare_hashmap(&rechecks_completed)) @@ -830,7 +854,7 @@ impl PendingPayableScanner { debug!( logger, - "Found {} pending payables and {} unfinalized failures to process", + "Found {} pending payables and {} suspected failures to process", resolve_optional_vec(pending_tx_hashes_opt), resolve_optional_vec(failure_hashes_opt) ); @@ -905,7 +929,7 @@ mod tests { .build(); let logger = Logger::new("start_scan_fills_in_caches_and_returns_msg"); let pending_payable_cache_before = subject.current_sent_payables.dump_cache(); - let failed_payable_cache_before = subject.yet_unproven_failed_payables.dump_cache(); + let failed_payable_cache_before = subject.suspected_failed_payables.dump_cache(); let result = subject.start_scan(&make_wallet("blah"), SystemTime::now(), None, &logger); @@ -932,7 +956,7 @@ mod tests { failed_payable_cache_before ); let pending_payable_cache_after = subject.current_sent_payables.dump_cache(); - let failed_payable_cache_after = subject.yet_unproven_failed_payables.dump_cache(); + let failed_payable_cache_after = subject.suspected_failed_payables.dump_cache(); assert_eq!( pending_payable_cache_after, hashmap!(sent_tx_hash_1 => sent_tx_1, sent_tx_hash_2 => sent_tx_2) @@ -1040,7 +1064,7 @@ mod tests { failed_payable_cache.load_cache(vec![failed_tx_1, failed_tx_2]); let mut subject = PendingPayableScannerBuilder::new().build(); subject.current_sent_payables = Box::new(pending_payable_cache); - subject.yet_unproven_failed_payables = Box::new(failed_payable_cache); + subject.suspected_failed_payables = Box::new(failed_payable_cache); let logger = Logger::new("test"); let msg = TxReceiptsMessage { results: btreemap![TxHashByTable::SentPayable(sent_tx_hash_1) => Ok( @@ -1061,7 +1085,7 @@ mod tests { values. Pending payables: [SentTx { hash: 0x0000000000000000000000000000000000000000000000\ 000000000000000890, receiver_address: 0x0000000000000000000558000000000558000000, \ amount_minor: 43237380096, timestamp: 29942784, gas_price_minor: 94818816, nonce: 456, \ - status: Pending(Waiting) }]. Unproven failures: []."; + status: Pending(Waiting) }]. Suspected failures: []."; assert_eq!(panic_msg, expected); } @@ -1080,7 +1104,7 @@ mod tests { failed_payable_cache.load_cache(vec![failed_tx_1]); let mut subject = PendingPayableScannerBuilder::new().build(); subject.current_sent_payables = Box::new(pending_payable_cache); - subject.yet_unproven_failed_payables = Box::new(failed_payable_cache); + subject.suspected_failed_payables = Box::new(failed_payable_cache); let logger = Logger::new("test"); let msg = TxReceiptsMessage { results: btreemap![TxHashByTable::SentPayable(sent_tx_hash_1) => Ok(StatusReadFromReceiptCheck::Pending), @@ -1103,7 +1127,7 @@ mod tests { Pending(Waiting) }, SentTx { hash: 0x0000000000000000000000000000000000000000000000000000000\ 000000315, receiver_address: 0x000000000000000000093f00000000093f000000, amount_minor: \ 387532395441, timestamp: 89643024, gas_price_minor: 491169069, nonce: 789, status: \ - Pending(Waiting) }]. Unproven failures: []."; + Pending(Waiting) }]. Suspected failures: []."; assert_eq!(panic_msg, expected); } @@ -1720,7 +1744,7 @@ mod tests { subject.handle_confirmed_transactions( DetectedConfirmations { - normal_confirmations: vec![], + standard_confirmations: vec![], reclaims: vec![sent_tx_1.clone(), sent_tx_2.clone()], }, &logger, @@ -1785,7 +1809,7 @@ mod tests { subject.handle_confirmed_transactions( DetectedConfirmations { - normal_confirmations: vec![], + standard_confirmations: vec![], reclaims: vec![sent_tx_1.clone(), sent_tx_2.clone()], }, &Logger::new("test"), @@ -1832,7 +1856,7 @@ mod tests { subject.handle_confirmed_transactions( DetectedConfirmations { - normal_confirmations: vec![], + standard_confirmations: vec![], reclaims: vec![sent_tx_1.clone(), sent_tx_2.clone()], }, &Logger::new("test"), @@ -1854,7 +1878,7 @@ mod tests { subject.handle_confirmed_transactions( DetectedConfirmations { - normal_confirmations: vec![], + standard_confirmations: vec![], reclaims: vec![sent_tx.clone()], }, &Logger::new("test"), @@ -1862,9 +1886,9 @@ mod tests { } #[test] - fn handles_normal_confirmations_alone() { + fn handles_standard_confirmations_alone() { init_test_logging(); - let test_name = "handles_normal_confirmations_alone"; + let test_name = "handles_standard_confirmations_alone"; let transactions_confirmed_params_arc = Arc::new(Mutex::new(vec![])); let confirm_tx_params_arc = Arc::new(Mutex::new(vec![])); let payable_dao = PayableDaoMock::default() @@ -1905,7 +1929,7 @@ mod tests { subject.handle_confirmed_transactions( DetectedConfirmations { - normal_confirmations: vec![sent_tx_1.clone(), sent_tx_2.clone()], + standard_confirmations: vec![sent_tx_1.clone(), sent_tx_2.clone()], reclaims: vec![], }, &logger, @@ -1981,7 +2005,7 @@ mod tests { subject.handle_confirmed_transactions( DetectedConfirmations { - normal_confirmations: vec![sent_tx_1.clone()], + standard_confirmations: vec![sent_tx_1.clone()], reclaims: vec![sent_tx_2.clone()], }, &logger, @@ -2045,7 +2069,7 @@ mod tests { subject.handle_confirmed_transactions( DetectedConfirmations { - normal_confirmations: vec![sent_tx_1, sent_tx_2], + standard_confirmations: vec![sent_tx_1, sent_tx_2], reclaims: vec![], }, &Logger::new("test"), @@ -2071,7 +2095,7 @@ mod tests { subject.handle_confirmed_transactions( DetectedConfirmations { - normal_confirmations: vec![sent_tx], + standard_confirmations: vec![sent_tx], reclaims: vec![], }, &Logger::new("test"), @@ -2153,7 +2177,7 @@ mod tests { subject.handle_confirmed_transactions( DetectedConfirmations { - normal_confirmations: vec![sent_tx_1, sent_tx_2], + standard_confirmations: vec![sent_tx_1, sent_tx_2], reclaims: vec![sent_tx_3], }, &Logger::new(test_name), diff --git a/node/src/accountant/scanners/pending_payable_scanner/tx_receipt_interpreter.rs b/node/src/accountant/scanners/pending_payable_scanner/tx_receipt_interpreter.rs index fc16c5713..d04d458b4 100644 --- a/node/src/accountant/scanners/pending_payable_scanner/tx_receipt_interpreter.rs +++ b/node/src/accountant/scanners/pending_payable_scanner/tx_receipt_interpreter.rs @@ -31,6 +31,7 @@ impl TxReceiptInterpreter { pending_payable_scanner: &PendingPayableScanner, logger: &Logger, ) -> ReceiptScanReport { + debug!(logger, "Composing receipt scan report"); let scan_report = ReceiptScanReport::default(); tx_cases .into_iter() @@ -162,7 +163,7 @@ impl TxReceiptInterpreter { scan_report } - //TODO: failures handling might need enhancement suggested by GH-693 + //TODO: if wanted, address GH-693 for more detailed failures fn handle_reverted_tx( mut scan_report: ReceiptScanReport, tx: TxByTable, @@ -185,7 +186,7 @@ impl TxReceiptInterpreter { failed_tx.reason, ); - scan_report.register_finalization_of_unproven_failure(failed_tx.hash); + scan_report.register_finalization_of_suspected_failure(failed_tx.hash); } } scan_report @@ -279,7 +280,7 @@ mod tests { ReceiptScanReport { failures: DetectedFailures::default(), confirmations: DetectedConfirmations { - normal_confirmations: vec![updated_tx], + standard_confirmations: vec![updated_tx], reclaims: vec![] } } @@ -327,7 +328,7 @@ mod tests { ReceiptScanReport { failures: DetectedFailures::default(), confirmations: DetectedConfirmations { - normal_confirmations: vec![], + standard_confirmations: vec![], reclaims: vec![sent_tx] } } diff --git a/node/src/accountant/scanners/pending_payable_scanner/utils.rs b/node/src/accountant/scanners/pending_payable_scanner/utils.rs index 7a1d18eaa..2b8052496 100644 --- a/node/src/accountant/scanners/pending_payable_scanner/utils.rs +++ b/node/src/accountant/scanners/pending_payable_scanner/utils.rs @@ -38,7 +38,9 @@ impl ReceiptScanReport { confirmation_type: ConfirmationType, ) { match confirmation_type { - ConfirmationType::Normal => self.confirmations.normal_confirmations.push(confirmed_tx), + ConfirmationType::Normal => { + self.confirmations.standard_confirmations.push(confirmed_tx) + } ConfirmationType::Reclaim => self.confirmations.reclaims.push(confirmed_tx), } } @@ -49,7 +51,7 @@ impl ReceiptScanReport { .push(PresortedTxFailure::NewEntry(failed_tx)); } - pub(super) fn register_finalization_of_unproven_failure(&mut self, tx_hash: TxHash) { + pub(super) fn register_finalization_of_suspected_failure(&mut self, tx_hash: TxHash) { self.failures .tx_failures .push(PresortedTxFailure::RecheckCompleted(tx_hash)); @@ -62,13 +64,13 @@ impl ReceiptScanReport { #[derive(Debug, Default, PartialEq, Eq, Clone)] pub struct DetectedConfirmations { - pub normal_confirmations: Vec, + pub standard_confirmations: Vec, pub reclaims: Vec, } impl DetectedConfirmations { pub(super) fn is_empty(&self) -> bool { - self.normal_confirmations.is_empty() && self.reclaims.is_empty() + self.standard_confirmations.is_empty() && self.reclaims.is_empty() } } @@ -410,7 +412,7 @@ mod tests { #[test] fn detected_confirmations_is_empty_works() { let subject = DetectedConfirmations { - normal_confirmations: vec![], + standard_confirmations: vec![], reclaims: vec![], }; @@ -462,19 +464,19 @@ mod tests { ]; let detected_confirmations_feeding = vec![ DetectedConfirmations { - normal_confirmations: vec![], + standard_confirmations: vec![], reclaims: vec![], }, DetectedConfirmations { - normal_confirmations: vec![make_sent_tx(456)], + standard_confirmations: vec![make_sent_tx(456)], reclaims: vec![make_sent_tx(999)], }, DetectedConfirmations { - normal_confirmations: vec![make_sent_tx(777)], + standard_confirmations: vec![make_sent_tx(777)], reclaims: vec![], }, DetectedConfirmations { - normal_confirmations: vec![], + standard_confirmations: vec![], reclaims: vec![make_sent_tx(999)], }, ]; @@ -550,19 +552,19 @@ mod tests { ]; let detected_confirmations_feeding = vec![ DetectedConfirmations { - normal_confirmations: vec![], + standard_confirmations: vec![], reclaims: vec![], }, DetectedConfirmations { - normal_confirmations: vec![make_sent_tx(777)], + standard_confirmations: vec![make_sent_tx(777)], reclaims: vec![make_sent_tx(999)], }, DetectedConfirmations { - normal_confirmations: vec![make_sent_tx(777)], + standard_confirmations: vec![make_sent_tx(777)], reclaims: vec![], }, DetectedConfirmations { - normal_confirmations: vec![], + standard_confirmations: vec![], reclaims: vec![make_sent_tx(999)], }, ]; @@ -594,15 +596,15 @@ mod tests { fn requires_payments_retry_says_no() { let detected_confirmations_feeding = vec![ DetectedConfirmations { - normal_confirmations: vec![make_sent_tx(777)], + standard_confirmations: vec![make_sent_tx(777)], reclaims: vec![make_sent_tx(999)], }, DetectedConfirmations { - normal_confirmations: vec![make_sent_tx(777)], + standard_confirmations: vec![make_sent_tx(777)], reclaims: vec![], }, DetectedConfirmations { - normal_confirmations: vec![], + standard_confirmations: vec![], reclaims: vec![make_sent_tx(999)], }, ]; @@ -638,7 +640,7 @@ mod tests { tx_receipt_rpc_failures: vec![], }, confirmations: DetectedConfirmations { - normal_confirmations: vec![], + standard_confirmations: vec![], reclaims: vec![], }, }; diff --git a/node/src/accountant/scanners/scan_schedulers.rs b/node/src/accountant/scanners/scan_schedulers.rs index dedfee1e5..5de151c73 100644 --- a/node/src/accountant/scanners/scan_schedulers.rs +++ b/node/src/accountant/scanners/scan_schedulers.rs @@ -14,6 +14,7 @@ use masq_lib::logger::Logger; use masq_lib::messages::ScanType; use masq_lib::simple_clock::{SimpleClock, SimpleClockReal}; use std::fmt::{Debug, Display, Formatter}; +use std::ops::Div; use std::time::{Duration, SystemTime, UNIX_EPOCH}; pub struct ScanSchedulers { @@ -50,28 +51,31 @@ pub enum ScanReschedulingAfterEarlyStop { } #[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum PayableSequenceScanner { +pub enum UnableToStartScanner { NewPayables, RetryPayables, PendingPayables { initial_pending_payable_scan: bool }, + Receivables, } -impl Display for PayableSequenceScanner { +impl Display for UnableToStartScanner { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - PayableSequenceScanner::NewPayables => write!(f, "NewPayables"), - PayableSequenceScanner::RetryPayables => write!(f, "RetryPayables"), - PayableSequenceScanner::PendingPayables { .. } => write!(f, "PendingPayables"), + UnableToStartScanner::NewPayables => write!(f, "NewPayables"), + UnableToStartScanner::RetryPayables => write!(f, "RetryPayables"), + UnableToStartScanner::PendingPayables { .. } => write!(f, "PendingPayables"), + UnableToStartScanner::Receivables => write!(f, "Receivables"), } } } -impl From for ScanType { - fn from(scanner: PayableSequenceScanner) -> Self { +impl From for ScanType { + fn from(scanner: UnableToStartScanner) -> Self { match scanner { - PayableSequenceScanner::NewPayables => ScanType::Payables, - PayableSequenceScanner::RetryPayables => ScanType::Payables, - PayableSequenceScanner::PendingPayables { .. } => ScanType::PendingPayables, + UnableToStartScanner::NewPayables => ScanType::Payables, + UnableToStartScanner::RetryPayables => ScanType::Payables, + UnableToStartScanner::PendingPayables { .. } => ScanType::PendingPayables, + UnableToStartScanner::Receivables => ScanType::Receivables, } } } @@ -80,23 +84,25 @@ pub struct PayableScanScheduler { pub new_payable_notify_later: Box>, pub interval_computer: Box, pub new_payable_notify: Box>, - pub retry_payable_notify: Box>, + pub retry_payable_notify_later: Box>, + pub retry_payable_scan_interval: Duration, } impl PayableScanScheduler { - fn new(new_payable_interval: Duration) -> Self { + fn new(payable_scan_interval: Duration) -> Self { Self { new_payable_notify_later: Box::new(NotifyLaterHandleReal::default()), interval_computer: Box::new(NewPayableScanIntervalComputerReal::new( - new_payable_interval, + payable_scan_interval, )), new_payable_notify: Box::new(NotifyHandleReal::default()), - retry_payable_notify: Box::new(NotifyHandleReal::default()), + retry_payable_notify_later: Box::new(NotifyLaterHandleReal::default()), + retry_payable_scan_interval: payable_scan_interval.div(2), } } pub fn schedule_new_payable_scan(&self, ctx: &mut Context, logger: &Logger) { - if let ScanTiming::WaitFor(interval) = self.interval_computer.time_until_next_scan() { + if let ScanTiming::WaitFor(interval) = self.interval_computer.compute_time_to_next_scan() { debug!( logger, "Scheduling a new-payable scan in {}ms", @@ -122,7 +128,8 @@ impl PayableScanScheduler { } } - pub fn reset_scan_timer(&mut self) { + pub fn reset_scan_timer(&mut self, logger: &Logger) { + debug!(logger, "NewPayableScanIntervalComputer timer reset"); self.interval_computer.reset_last_scan_timestamp(); } @@ -136,18 +143,20 @@ impl PayableScanScheduler { logger: &Logger, ) { debug!(logger, "Scheduling a retry-payable scan asap"); + let delay = self.retry_payable_scan_interval; - self.retry_payable_notify.notify( + let _ = self.retry_payable_notify_later.notify_later( ScanForRetryPayables { response_skeleton_opt, }, + delay, ctx, - ) + ); } } pub trait NewPayableScanIntervalComputer { - fn time_until_next_scan(&self) -> ScanTiming; + fn compute_time_to_next_scan(&self) -> ScanTiming; fn reset_last_scan_timestamp(&mut self); @@ -161,7 +170,7 @@ pub struct NewPayableScanIntervalComputerReal { } impl NewPayableScanIntervalComputer for NewPayableScanIntervalComputerReal { - fn time_until_next_scan(&self) -> ScanTiming { + fn compute_time_to_next_scan(&self) -> ScanTiming { let current_time = self.clock.now(); let time_since_last_scan = current_time .duration_since(self.last_scan_timestamp) @@ -244,7 +253,7 @@ where pub trait RescheduleScanOnErrorResolver { fn resolve_rescheduling_on_error( &self, - scanner: PayableSequenceScanner, + scanner: UnableToStartScanner, error: &StartScanError, is_externally_triggered: bool, logger: &Logger, @@ -257,25 +266,28 @@ pub struct RescheduleScanOnErrorResolverReal {} impl RescheduleScanOnErrorResolver for RescheduleScanOnErrorResolverReal { fn resolve_rescheduling_on_error( &self, - scanner: PayableSequenceScanner, + scanner: UnableToStartScanner, error: &StartScanError, is_externally_triggered: bool, logger: &Logger, ) -> ScanReschedulingAfterEarlyStop { let reschedule_hint = match scanner { - PayableSequenceScanner::NewPayables => { + UnableToStartScanner::NewPayables => { Self::resolve_new_payables(error, is_externally_triggered) } - PayableSequenceScanner::RetryPayables => { + UnableToStartScanner::RetryPayables => { Self::resolve_retry_payables(error, is_externally_triggered) } - PayableSequenceScanner::PendingPayables { + UnableToStartScanner::PendingPayables { initial_pending_payable_scan, } => Self::resolve_pending_payables( error, initial_pending_payable_scan, is_externally_triggered, ), + UnableToStartScanner::Receivables => { + Self::resolve_receivables(error, is_externally_triggered) + } }; Self::log_rescheduling(scanner, is_externally_triggered, logger, &reschedule_hint); @@ -290,14 +302,18 @@ impl RescheduleScanOnErrorResolverReal { is_externally_triggered: bool, ) -> ScanReschedulingAfterEarlyStop { if is_externally_triggered { - ScanReschedulingAfterEarlyStop::DoNotSchedule - } else if matches!(err, StartScanError::ScanAlreadyRunning { .. }) { - unreachable!( - "an automatic scan of NewPayableScanner should never interfere with itself {:?}", - err - ) - } else { - ScanReschedulingAfterEarlyStop::Schedule(ScanType::Payables) + return ScanReschedulingAfterEarlyStop::DoNotSchedule; + } + + match err { + StartScanError::CalledFromNullScanner => ScanReschedulingAfterEarlyStop::DoNotSchedule, + StartScanError::ScanAlreadyRunning { .. } => { + unreachable!( + "an automatic scan of NewPayableScanner should never interfere with itself {:?}", + err + ) + } + _ => ScanReschedulingAfterEarlyStop::Schedule(ScanType::Payables), } } @@ -324,18 +340,26 @@ impl RescheduleScanOnErrorResolverReal { is_externally_triggered: bool, ) -> ScanReschedulingAfterEarlyStop { if is_externally_triggered { - ScanReschedulingAfterEarlyStop::DoNotSchedule - } else if err == &StartScanError::NothingToProcess { - if initial_pending_payable_scan { + return ScanReschedulingAfterEarlyStop::DoNotSchedule; + } + + match err { + StartScanError::NothingToProcess => { + if !initial_pending_payable_scan { + unreachable!( + "the automatic pending payable scan should always be requested only in need, \ + which contradicts the current StartScanError::NothingToProcess" + ) + } ScanReschedulingAfterEarlyStop::Schedule(ScanType::Payables) - } else { - unreachable!( - "the automatic pending payable scan should always be requested only in need, \ - which contradicts the current StartScanError::NothingToProcess" - ) } - } else if err == &StartScanError::NoConsumingWalletFound { - if initial_pending_payable_scan { + StartScanError::NoConsumingWalletFound => { + if !initial_pending_payable_scan { + unreachable!( + "PendingPayableScanner called later than the initial attempt, but \ + the consuming wallet is still missing; this should not be possible" + ) + } // Cannot deduce there are strayed pending payables from the previous Node's run // (StartScanError::NoConsumingWalletFound is thrown before // StartScanError::NothingToProcess can be evaluated); but may be cautious and @@ -344,22 +368,34 @@ impl RescheduleScanOnErrorResolverReal { // TODO Correctly, a check-point during the bootstrap, not allowing to come // this far, should be the solution. Part of the issue mentioned in GH-799 ScanReschedulingAfterEarlyStop::Schedule(ScanType::PendingPayables) - } else { - unreachable!( - "PendingPayableScanner called later than the initial attempt, but \ - the consuming wallet is still missing; this should not be possible" - ) } - } else { - unreachable!( + StartScanError::CalledFromNullScanner => ScanReschedulingAfterEarlyStop::DoNotSchedule, + _ => unreachable!( "{:?} should be impossible with PendingPayableScanner in automatic mode", err - ) + ), + } + } + + fn resolve_receivables( + err: &StartScanError, + is_externally_triggered: bool, + ) -> ScanReschedulingAfterEarlyStop { + if is_externally_triggered { + return ScanReschedulingAfterEarlyStop::DoNotSchedule; + } + + match err { + StartScanError::CalledFromNullScanner => ScanReschedulingAfterEarlyStop::DoNotSchedule, + _ => unreachable!( + "{:?} should be impossible with ReceivableScanner in automatic mode", + err + ), } } fn log_rescheduling( - scanner: PayableSequenceScanner, + scanner: UnableToStartScanner, is_externally_triggered: bool, logger: &Logger, reschedule_hint: &ScanReschedulingAfterEarlyStop, @@ -383,8 +419,8 @@ impl RescheduleScanOnErrorResolverReal { #[cfg(test)] mod tests { use crate::accountant::scanners::scan_schedulers::{ - NewPayableScanIntervalComputer, NewPayableScanIntervalComputerReal, PayableSequenceScanner, - ScanReschedulingAfterEarlyStop, ScanSchedulers, ScanTiming, + NewPayableScanIntervalComputer, NewPayableScanIntervalComputerReal, + ScanReschedulingAfterEarlyStop, ScanSchedulers, ScanTiming, UnableToStartScanner, }; use crate::accountant::scanners::test_utils::NewPayableScanIntervalComputerMock; use crate::accountant::scanners::{ManulTriggerError, StartScanError}; @@ -463,7 +499,7 @@ mod tests { subject.scan_interval = standard_interval; subject.last_scan_timestamp = past_instant; - let result = subject.time_until_next_scan(); + let result = subject.compute_time_to_next_scan(); assert_eq!( result, @@ -498,7 +534,7 @@ mod tests { subject.scan_interval = standard_interval; subject.last_scan_timestamp = past_instant; - let result = subject.time_until_next_scan(); + let result = subject.compute_time_to_next_scan(); assert_eq!( result, @@ -518,7 +554,7 @@ mod tests { subject.scan_interval = Duration::from_secs(180); subject.clock = Box::new(SimpleClockMock::default().now_result(now)); - let result = subject.time_until_next_scan(); + let result = subject.compute_time_to_next_scan(); assert_eq!( result, @@ -564,7 +600,7 @@ mod tests { subject.clock = Box::new(SimpleClockMock::default().now_result(now)); subject.last_scan_timestamp = now.checked_add(Duration::from_secs(1)).unwrap(); - let _ = subject.time_until_next_scan(); + let _ = subject.compute_time_to_next_scan(); } #[test] @@ -612,7 +648,7 @@ mod tests { .reset_last_scan_timestamp_params(&reset_last_scan_timestamp_params_arc), ); - subject.payable.reset_scan_timer(); + subject.payable.reset_scan_timer(&Logger::new("test")); let reset_last_scan_timestamp_params = reset_last_scan_timestamp_params_arc.lock().unwrap(); assert_eq!(*reset_last_scan_timestamp_params, vec![()]) @@ -642,7 +678,6 @@ mod tests { StartScanError::CalledFromNullScanner ]; - let mut check_vec = candidates .iter() .fold(vec![],|mut acc, current|{ @@ -703,14 +738,14 @@ mod tests { test_what_if_externally_triggered( &format!("{}(initial_pending_payable_scan = false)", test_name), &subject, - PayableSequenceScanner::PendingPayables { + UnableToStartScanner::PendingPayables { initial_pending_payable_scan: false, }, ); test_what_if_externally_triggered( &format!("{}(initial_pending_payable_scan = true)", test_name), &subject, - PayableSequenceScanner::PendingPayables { + UnableToStartScanner::PendingPayables { initial_pending_payable_scan: true, }, ); @@ -719,7 +754,7 @@ mod tests { fn test_what_if_externally_triggered( test_name: &str, subject: &ScanSchedulers, - scanner: PayableSequenceScanner, + scanner: UnableToStartScanner, ) { init_test_logging(); let logger = Logger::new(test_name); @@ -749,17 +784,16 @@ mod tests { } #[test] - fn resolve_error_for_pending_payables_if_nothing_to_process_and_initial_pending_payable_scan_true( - ) { + fn resolve_error_if_nothing_to_process_and_initial_pending_payable_scan_true() { init_test_logging(); let subject = ScanSchedulers::new(*TEST_SCAN_INTERVALS, true); - let test_name = "resolve_error_for_pending_payables_if_nothing_to_process_and_initial_pending_payable_scan_true"; + let test_name = "resolve_error_if_nothing_to_process_and_initial_pending_payable_scan_true"; let logger = Logger::new(test_name); let result = subject .reschedule_on_error_resolver .resolve_rescheduling_on_error( - PayableSequenceScanner::PendingPayables { + UnableToStartScanner::PendingPayables { initial_pending_payable_scan: true, }, &StartScanError::NothingToProcess, @@ -792,7 +826,7 @@ mod tests { let _ = subject .reschedule_on_error_resolver .resolve_rescheduling_on_error( - PayableSequenceScanner::PendingPayables { + UnableToStartScanner::PendingPayables { initial_pending_payable_scan: false, }, &StartScanError::NothingToProcess, @@ -802,12 +836,13 @@ mod tests { } #[test] - fn resolve_error_for_pending_p_if_no_consuming_wallet_found_in_initial_pending_payable_scan() { + fn resolve_error_if_no_consuming_wallet_found_in_initial_pending_payable_scan() { init_test_logging(); - let test_name = "resolve_error_for_pending_p_if_no_consuming_wallet_found_in_initial_pending_payable_scan"; + let test_name = + "resolve_error_if_no_consuming_wallet_found_in_initial_pending_payable_scan"; let logger = Logger::new(test_name); let subject = ScanSchedulers::new(*TEST_SCAN_INTERVALS, true); - let scanner = PayableSequenceScanner::PendingPayables { + let scanner = UnableToStartScanner::PendingPayables { initial_pending_payable_scan: true, }; @@ -839,9 +874,10 @@ mod tests { than the initial attempt, but the consuming wallet is still missing; this should not be \ possible" )] - fn pending_p_scan_attempt_if_no_consuming_wallet_found_mustnt_happen_if_not_initial_scan() { + fn pending_payable_scan_attempt_if_no_consuming_wallet_found_mustnt_happen_if_not_initial_scan() + { let subject = ScanSchedulers::new(*TEST_SCAN_INTERVALS, true); - let scanner = PayableSequenceScanner::PendingPayables { + let scanner = UnableToStartScanner::PendingPayables { initial_pending_payable_scan: false, }; @@ -855,6 +891,20 @@ mod tests { ); } + #[test] + fn resolve_error_for_pending_payables_if_null_scanner_is_used_in_automatic_mode() { + test_resolve_error_if_null_scanner_is_used_in_automatic_mode( + UnableToStartScanner::PendingPayables { + initial_pending_payable_scan: true, + }, + ); + test_resolve_error_if_null_scanner_is_used_in_automatic_mode( + UnableToStartScanner::PendingPayables { + initial_pending_payable_scan: false, + }, + ); + } + #[test] fn resolve_error_for_pending_payables_forbidden_states() { fn test_forbidden_states( @@ -867,7 +917,7 @@ mod tests { subject .reschedule_on_error_resolver .resolve_rescheduling_on_error( - PayableSequenceScanner::PendingPayables { + UnableToStartScanner::PendingPayables { initial_pending_payable_scan, }, *error, @@ -894,6 +944,7 @@ mod tests { let inputs = ListOfStartScanErrors::default().eliminate_already_tested_variants(vec![ StartScanError::NothingToProcess, StartScanError::NoConsumingWalletFound, + StartScanError::CalledFromNullScanner, ]); let subject = ScanSchedulers::new(*TEST_SCAN_INTERVALS, true); @@ -910,7 +961,7 @@ mod tests { test_what_if_externally_triggered( test_name, &subject, - PayableSequenceScanner::RetryPayables {}, + UnableToStartScanner::RetryPayables {}, ); } @@ -923,7 +974,7 @@ mod tests { subject .reschedule_on_error_resolver .resolve_rescheduling_on_error( - PayableSequenceScanner::RetryPayables, + UnableToStartScanner::RetryPayables, error, false, &Logger::new("test"), @@ -946,15 +997,14 @@ mod tests { } #[test] - fn resolve_rescheduling_on_error_works_for_new_payables_if_externally_triggered() { - let test_name = - "resolve_rescheduling_on_error_works_for_new_payables_if_externally_triggered"; + fn resolve_rescheduling_on_error_for_new_payables_if_externally_triggered() { + let test_name = "resolve_rescheduling_on_error_for_new_payables_if_externally_triggered"; let subject = ScanSchedulers::new(*TEST_SCAN_INTERVALS, true); test_what_if_externally_triggered( test_name, &subject, - PayableSequenceScanner::NewPayables {}, + UnableToStartScanner::NewPayables {}, ); } @@ -963,13 +1013,13 @@ mod tests { expected = "internal error: entered unreachable code: an automatic scan of NewPayableScanner \ should never interfere with itself ScanAlreadyRunning { cross_scan_cause_opt: None, started_at:" )] - fn resolve_hint_for_new_payables_if_scan_is_already_running_error_and_is_automatic_scan() { + fn resolve_hint_for_new_payables_error_if_scan_is_already_running_in_automatic_scan() { let subject = ScanSchedulers::new(*TEST_SCAN_INTERVALS, true); let _ = subject .reschedule_on_error_resolver .resolve_rescheduling_on_error( - PayableSequenceScanner::NewPayables, + UnableToStartScanner::NewPayables, &StartScanError::ScanAlreadyRunning { cross_scan_cause_opt: None, started_at: SystemTime::now(), @@ -979,10 +1029,18 @@ mod tests { ); } + #[test] + fn resolve_error_for_new_payables_if_null_scanner_is_used_in_automatic_mode() { + test_resolve_error_if_null_scanner_is_used_in_automatic_mode( + UnableToStartScanner::NewPayables, + ); + } + #[test] fn resolve_new_payables_with_error_cases_resulting_in_future_rescheduling() { let test_name = "resolve_new_payables_with_error_cases_resulting_in_future_rescheduling"; let inputs = ListOfStartScanErrors::default().eliminate_already_tested_variants(vec![ + StartScanError::CalledFromNullScanner, StartScanError::ScanAlreadyRunning { cross_scan_cause_opt: None, started_at: SystemTime::now(), @@ -996,7 +1054,7 @@ mod tests { let result = subject .reschedule_on_error_resolver .resolve_rescheduling_on_error( - PayableSequenceScanner::NewPayables, + UnableToStartScanner::NewPayables, *error, false, &logger, @@ -1015,27 +1073,100 @@ mod tests { }) } + #[test] + fn resolve_rescheduling_on_error_for_receivables_if_externally_triggered() { + let test_name = "resolve_rescheduling_on_error_for_receivables_if_externally_triggered"; + let subject = ScanSchedulers::new(*TEST_SCAN_INTERVALS, true); + + test_what_if_externally_triggered(test_name, &subject, UnableToStartScanner::Receivables); + } + + #[test] + fn resolve_error_for_receivables_if_null_scanner_is_used_in_automatic_mode() { + test_resolve_error_if_null_scanner_is_used_in_automatic_mode( + UnableToStartScanner::Receivables, + ); + } + + #[test] + fn resolve_error_for_receivables_all_fatal_cases_in_automatic_mode() { + let inputs = ListOfStartScanErrors::default() + .eliminate_already_tested_variants(vec![StartScanError::CalledFromNullScanner]); + let subject = ScanSchedulers::new(*TEST_SCAN_INTERVALS, true); + + inputs.errors.iter().for_each(|error| { + let panic = catch_unwind(AssertUnwindSafe(|| { + subject + .reschedule_on_error_resolver + .resolve_rescheduling_on_error( + UnableToStartScanner::Receivables, + *error, + false, + &Logger::new("test"), + ) + })) + .unwrap_err(); + + let panic_msg = panic.downcast_ref::().unwrap(); + let expected_msg = format!( + "internal error: entered unreachable code: {:?} should be impossible with \ + ReceivableScanner in automatic mode", + error + ); + assert_eq!( + panic_msg, &expected_msg, + "We expected '{}' but got '{}'", + expected_msg, panic_msg + ) + }) + } + #[test] fn conversion_between_hintable_scanner_and_scan_type_works() { assert_eq!( - ScanType::from(PayableSequenceScanner::NewPayables), + ScanType::from(UnableToStartScanner::NewPayables), ScanType::Payables ); assert_eq!( - ScanType::from(PayableSequenceScanner::RetryPayables), + ScanType::from(UnableToStartScanner::RetryPayables), ScanType::Payables ); assert_eq!( - ScanType::from(PayableSequenceScanner::PendingPayables { + ScanType::from(UnableToStartScanner::PendingPayables { initial_pending_payable_scan: false }), ScanType::PendingPayables ); assert_eq!( - ScanType::from(PayableSequenceScanner::PendingPayables { + ScanType::from(UnableToStartScanner::PendingPayables { initial_pending_payable_scan: true }), ScanType::PendingPayables ); + assert_eq!( + ScanType::from(UnableToStartScanner::Receivables), + ScanType::Receivables + ) + } + + fn test_resolve_error_if_null_scanner_is_used_in_automatic_mode(scanner: UnableToStartScanner) { + let subject = ScanSchedulers::new(*TEST_SCAN_INTERVALS, true); + + let result = subject + .reschedule_on_error_resolver + .resolve_rescheduling_on_error( + scanner, + &StartScanError::CalledFromNullScanner, + false, + &Logger::new("test"), + ); + + assert_eq!( + result, + ScanReschedulingAfterEarlyStop::DoNotSchedule, + "We expected DoNotSchedule but got {:?} for {:?}", + result, + scanner + ); } } diff --git a/node/src/accountant/scanners/test_utils.rs b/node/src/accountant/scanners/test_utils.rs index 08325dedc..031e1bb7d 100644 --- a/node/src/accountant/scanners/test_utils.rs +++ b/node/src/accountant/scanners/test_utils.rs @@ -18,8 +18,8 @@ use crate::accountant::scanners::pending_payable_scanner::{ CachesEmptiableScanner, ExtendedPendingPayablePrivateScanner, }; use crate::accountant::scanners::scan_schedulers::{ - NewPayableScanIntervalComputer, PayableSequenceScanner, RescheduleScanOnErrorResolver, - ScanReschedulingAfterEarlyStop, ScanTiming, + NewPayableScanIntervalComputer, RescheduleScanOnErrorResolver, ScanReschedulingAfterEarlyStop, + ScanTiming, UnableToStartScanner, }; use crate::accountant::scanners::{ PendingPayableScanner, PrivateScanner, RealScannerMarker, ReceivableScanner, Scanner, @@ -336,15 +336,20 @@ impl ScannerMockMarker for ScannerMock>>, - time_until_next_scan_results: RefCell>, + compute_time_to_next_scan_params: Arc>>, + compute_time_to_next_scan_results: RefCell>, reset_last_scan_timestamp_params: Arc>>, } impl NewPayableScanIntervalComputer for NewPayableScanIntervalComputerMock { - fn time_until_next_scan(&self) -> ScanTiming { - self.time_until_next_scan_params.lock().unwrap().push(()); - self.time_until_next_scan_results.borrow_mut().remove(0) + fn compute_time_to_next_scan(&self) -> ScanTiming { + self.compute_time_to_next_scan_params + .lock() + .unwrap() + .push(()); + self.compute_time_to_next_scan_results + .borrow_mut() + .remove(0) } fn reset_last_scan_timestamp(&mut self) { @@ -358,13 +363,15 @@ impl NewPayableScanIntervalComputer for NewPayableScanIntervalComputerMock { } impl NewPayableScanIntervalComputerMock { - pub fn time_until_next_scan_params(mut self, params: &Arc>>) -> Self { - self.time_until_next_scan_params = params.clone(); + pub fn compute_time_to_next_scan_params(mut self, params: &Arc>>) -> Self { + self.compute_time_to_next_scan_params = params.clone(); self } - pub fn time_until_next_scan_result(self, result: ScanTiming) -> Self { - self.time_until_next_scan_results.borrow_mut().push(result); + pub fn compute_time_to_next_scan_result(self, result: ScanTiming) -> Self { + self.compute_time_to_next_scan_results + .borrow_mut() + .push(result); self } @@ -470,14 +477,14 @@ pub fn assert_timestamps_from_str(examined_str: &str, expected_timestamps: Vec>>, + Arc>>, resolve_rescheduling_on_error_results: RefCell>, } impl RescheduleScanOnErrorResolver for RescheduleScanOnErrorResolverMock { fn resolve_rescheduling_on_error( &self, - scanner: PayableSequenceScanner, + scanner: UnableToStartScanner, error: &StartScanError, is_externally_triggered: bool, logger: &Logger, @@ -500,7 +507,7 @@ impl RescheduleScanOnErrorResolver for RescheduleScanOnErrorResolverMock { impl RescheduleScanOnErrorResolverMock { pub fn resolve_rescheduling_on_error_params( mut self, - params: &Arc>>, + params: &Arc>>, ) -> Self { self.resolve_rescheduling_on_error_params = params.clone(); self diff --git a/node/src/accountant/test_utils.rs b/node/src/accountant/test_utils.rs index b460363ee..9e3e06575 100644 --- a/node/src/accountant/test_utils.rs +++ b/node/src/accountant/test_utils.rs @@ -1281,7 +1281,7 @@ pub struct PendingPayableScannerBuilder { payment_thresholds: PaymentThresholds, financial_statistics: FinancialStatistics, current_sent_payables: Box>, - yet_unproven_failed_payables: Box>, + suspected_failed_payables: Box>, clock: Box, } @@ -1294,7 +1294,7 @@ impl PendingPayableScannerBuilder { payment_thresholds: PaymentThresholds::default(), financial_statistics: FinancialStatistics::default(), current_sent_payables: Box::new(PendingPayableCacheMock::default()), - yet_unproven_failed_payables: Box::new(PendingPayableCacheMock::default()), + suspected_failed_payables: Box::new(PendingPayableCacheMock::default()), clock: Box::new(SimpleClockMock::default()), } } @@ -1323,7 +1323,7 @@ impl PendingPayableScannerBuilder { mut self, failures: Box>, ) -> Self { - self.yet_unproven_failed_payables = failures; + self.suspected_failed_payables = failures; self } @@ -1341,7 +1341,7 @@ impl PendingPayableScannerBuilder { Rc::new(RefCell::new(self.financial_statistics)), ); scanner.current_sent_payables = self.current_sent_payables; - scanner.yet_unproven_failed_payables = self.yet_unproven_failed_payables; + scanner.suspected_failed_payables = self.suspected_failed_payables; scanner.clock = self.clock; scanner } diff --git a/node/src/actor_system_factory.rs b/node/src/actor_system_factory.rs index 6b811cb09..8d750a647 100644 --- a/node/src/actor_system_factory.rs +++ b/node/src/actor_system_factory.rs @@ -1142,6 +1142,7 @@ mod tests { data_directory: PathBuf::new(), node_descriptor: NodeDescriptor::default(), mapping_protocol_opt: None, + new_public_key_opt: None, real_user: RealUser::null(), neighborhood_config: NeighborhoodConfig { mode: NeighborhoodMode::Standard( @@ -1213,6 +1214,7 @@ mod tests { cryptde_pair: CRYPTDE_PAIR.clone(), mapping_protocol_opt: Some(Igdp), real_user: RealUser::null(), + new_public_key_opt: None, neighborhood_config: NeighborhoodConfig { mode: NeighborhoodMode::Standard( NodeAddr::new(&IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), &[1234, 2345]), @@ -1509,6 +1511,7 @@ mod tests { cryptde_pair: CRYPTDE_PAIR.clone(), mapping_protocol_opt: None, real_user: RealUser::null(), + new_public_key_opt: None, neighborhood_config: NeighborhoodConfig { mode: NeighborhoodMode::ConsumeOnly(vec![]), min_hops: MIN_HOPS_FOR_TEST, @@ -1692,6 +1695,7 @@ mod tests { cryptde_pair: CRYPTDE_PAIR.clone(), mapping_protocol_opt: None, real_user: RealUser::null(), + new_public_key_opt: None, neighborhood_config: NeighborhoodConfig { mode: NeighborhoodMode::Standard( NodeAddr::new(&IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4)), &[]), diff --git a/node/src/blockchain/blockchain_agent/agent_web3.rs b/node/src/blockchain/blockchain_agent/agent_web3.rs index 66df08d57..4446642cd 100644 --- a/node/src/blockchain/blockchain_agent/agent_web3.rs +++ b/node/src/blockchain/blockchain_agent/agent_web3.rs @@ -118,11 +118,13 @@ mod tests { BlockchainAgentWeb3, WEB3_MAXIMAL_GAS_LIMIT_MARGIN, }; use crate::blockchain::blockchain_agent::BlockchainAgent; - use crate::blockchain::blockchain_bridge::increase_gas_price_by_margin; + use crate::blockchain::blockchain_bridge::increase_by_percentage; use crate::test_utils::make_wallet; use itertools::{Either, Itertools}; use masq_lib::blockchains::chains::Chain; - use masq_lib::constants::DEFAULT_GAS_PRICE_MARGIN; + use masq_lib::constants::{ + DEFAULT_GAS_PRICE_RETRY_CONSTANT, DEFAULT_GAS_PRICE_RETRY_PERCENTAGE, + }; use masq_lib::logger::Logger; use masq_lib::test_utils::logging::{init_test_logging, TestLogHandler}; use masq_lib::test_utils::utils::TEST_DEFAULT_CHAIN; @@ -155,7 +157,7 @@ mod tests { let result = subject.price_qualified_payables(Either::Left(new_tx_templates.clone())); - let gas_price_with_margin_wei = increase_gas_price_by_margin(rpc_gas_price_wei); + let gas_price_with_margin_wei = increase_by_percentage(rpc_gas_price_wei); let expected_result = Either::Left(PricedNewTxTemplates::new( new_tx_templates, gas_price_with_margin_wei, @@ -170,30 +172,32 @@ mod tests { let test_name = "returns_correct_priced_qualified_payables_for_retry_payable_scan"; let consuming_wallet = make_wallet("efg"); let consuming_wallet_balances = make_zeroed_consuming_wallet_balances(); - let rpc_gas_price_wei = 444_555_666; + let latest_gas_price_wei = 444_555_666; let chain = TEST_DEFAULT_CHAIN; + let prev_gas_prices = vec![ + latest_gas_price_wei - 1, + latest_gas_price_wei, + latest_gas_price_wei + 1, + latest_gas_price_wei - 123_456, + latest_gas_price_wei + 456_789, + ]; let retry_tx_templates: Vec = { - vec![ - rpc_gas_price_wei - 1, - rpc_gas_price_wei, - rpc_gas_price_wei + 1, - rpc_gas_price_wei - 123_456, - rpc_gas_price_wei + 456_789, - ] - .into_iter() - .enumerate() - .map(|(idx, prev_gas_price_wei)| { - let account = make_payable_account((idx as u64 + 1) * 3_000); - RetryTxTemplate { - base: BaseTxTemplate::from(&account), - prev_gas_price_wei, - prev_nonce: idx as u64, - } - }) - .collect_vec() + prev_gas_prices + .iter() + .copied() + .enumerate() + .map(|(idx, prev_gas_price_wei)| { + let account = make_payable_account((idx as u64 + 1) * 3_000); + RetryTxTemplate { + base: BaseTxTemplate::from(&account), + prev_gas_price_wei, + prev_nonce: idx as u64, + } + }) + .collect_vec() }; let mut subject = BlockchainAgentWeb3::new( - rpc_gas_price_wei, + latest_gas_price_wei, 77_777, consuming_wallet, consuming_wallet_balances, @@ -205,23 +209,27 @@ mod tests { .price_qualified_payables(Either::Right(RetryTxTemplates(retry_tx_templates.clone()))); let expected_result = { - let price_wei_for_accounts_from_1_to_5 = vec![ - increase_gas_price_by_margin(rpc_gas_price_wei), - increase_gas_price_by_margin(rpc_gas_price_wei), - increase_gas_price_by_margin(rpc_gas_price_wei + 1), - increase_gas_price_by_margin(rpc_gas_price_wei), - increase_gas_price_by_margin(rpc_gas_price_wei + 456_789), - ]; - if price_wei_for_accounts_from_1_to_5.len() != retry_tx_templates.len() { + let computed_gas_prices = prev_gas_prices + .into_iter() + .map(|prev_gas_price_wei| { + PricedRetryTxTemplate::compute_gas_price( + latest_gas_price_wei, + prev_gas_price_wei, + ) + }) + .collect::>(); + if computed_gas_prices.len() != retry_tx_templates.len() { panic!("Corrupted test") } - Either::Right(PricedRetryTxTemplates( retry_tx_templates .iter() - .zip(price_wei_for_accounts_from_1_to_5.into_iter()) - .map(|(retry_tx_template, increased_gas_price)| { - PricedRetryTxTemplate::new(retry_tx_template.clone(), increased_gas_price) + .zip(computed_gas_prices.into_iter()) + .map(|(retry_tx_template, computed_gas_price_wei)| { + PricedRetryTxTemplate::new( + retry_tx_template.clone(), + computed_gas_price_wei, + ) }) .collect_vec(), )) @@ -238,9 +246,10 @@ mod tests { // This should be the value that would surplus the ceiling just slightly if the margin is // applied. // Adding just 1 didn't work, therefore 2 - let rpc_gas_price_wei = - ((ceiling_gas_price_wei * 100) / (DEFAULT_GAS_PRICE_MARGIN as u128 + 100)) + 2; - let check_value_wei = increase_gas_price_by_margin(rpc_gas_price_wei); + let rpc_gas_price_wei = ((ceiling_gas_price_wei * 100) + / (DEFAULT_GAS_PRICE_RETRY_PERCENTAGE as u128 + 100)) + + 2; + let check_value_wei = increase_by_percentage(rpc_gas_price_wei); test_gas_price_must_not_break_through_ceiling_value_in_the_new_payable_mode( test_name, @@ -340,8 +349,8 @@ mod tests { // applied. // Adding just 1 didn't work, therefore 2 let rpc_gas_price_wei = - (ceiling_gas_price_wei * 100) / (DEFAULT_GAS_PRICE_MARGIN as u128 + 100) + 2; - let check_value_wei = increase_gas_price_by_margin(rpc_gas_price_wei); + (ceiling_gas_price_wei * 100) / (DEFAULT_GAS_PRICE_RETRY_PERCENTAGE as u128 + 100) + 2; + let check_value_wei = increase_by_percentage(rpc_gas_price_wei); let template_1 = RetryTxTemplateBuilder::new() .payable_account(&account_1) .prev_gas_price_wei(rpc_gas_price_wei - 1) @@ -382,18 +391,17 @@ mod tests { let account_1 = make_payable_account(12); let account_2 = make_payable_account(34); let ceiling_gas_price_wei = chain.rec().gas_price_safe_ceiling_minor; - // This should be the value that would surplus the ceiling just slightly if the margin is applied - let border_gas_price_wei = - (ceiling_gas_price_wei * 100) / (DEFAULT_GAS_PRICE_MARGIN as u128 + 100) + 2; - let rpc_gas_price_wei = border_gas_price_wei - 1; - let check_value_wei = increase_gas_price_by_margin(border_gas_price_wei); + // Once the gas price is computed from latest and prev gas price values, it'll break the ceiling + let prev_gas_price_wei = ceiling_gas_price_wei + 1 - DEFAULT_GAS_PRICE_RETRY_CONSTANT; + let latest_gas_price_wei = prev_gas_price_wei - 1; + let check_value_wei = prev_gas_price_wei + DEFAULT_GAS_PRICE_RETRY_CONSTANT; let template_1 = RetryTxTemplateBuilder::new() .payable_account(&account_1) - .prev_gas_price_wei(border_gas_price_wei) + .prev_gas_price_wei(prev_gas_price_wei) .build(); let template_2 = RetryTxTemplateBuilder::new() .payable_account(&account_2) - .prev_gas_price_wei(border_gas_price_wei) + .prev_gas_price_wei(prev_gas_price_wei) .build(); let retry_tx_templates = vec![template_1, template_2]; let expected_log_msg = format!( @@ -406,7 +414,7 @@ mod tests { test_gas_price_must_not_break_through_ceiling_value_in_the_retry_payable_mode( test_name, chain, - rpc_gas_price_wei, + latest_gas_price_wei, Either::Right(RetryTxTemplates(retry_tx_templates)), &expected_log_msg, ); @@ -466,8 +474,8 @@ mod tests { let expected_log_msg = format!( "The computed gas price(s) in wei is above the ceil value of 50,000,000,000 wei computed by this Node.\n\ Transaction(s) to following receivers are affected:\n\ - 0x00000000000000000000000077616c6c65743132 with gas price 64,999,999,998\n\ - 0x00000000000000000000000077616c6c65743334 with gas price 64,999,999,997" + 0x00000000000000000000000077616c6c65743132 with gas price 50,000,004,999\n\ + 0x00000000000000000000000077616c6c65743334 with gas price 50,000,004,998" ); test_gas_price_must_not_break_through_ceiling_value_in_the_retry_payable_mode( @@ -602,8 +610,7 @@ mod tests { assert_eq!( result, - (2 * (77_777 + WEB3_MAXIMAL_GAS_LIMIT_MARGIN)) - * increase_gas_price_by_margin(444_555_666) + (2 * (77_777 + WEB3_MAXIMAL_GAS_LIMIT_MARGIN)) * increase_by_percentage(444_555_666) ); } @@ -611,30 +618,32 @@ mod tests { fn estimate_transaction_fee_total_works_for_retry_txs() { let consuming_wallet = make_wallet("efg"); let consuming_wallet_balances = make_zeroed_consuming_wallet_balances(); - let rpc_gas_price_wei = 444_555_666; + let latest_gas_price_wei = 444_555_666; let chain = TEST_DEFAULT_CHAIN; + let prev_gas_prices = vec![ + latest_gas_price_wei - 1, + latest_gas_price_wei, + latest_gas_price_wei + 1, + latest_gas_price_wei - 123_456, + latest_gas_price_wei + 456_789, + ]; let retry_tx_templates: Vec = { - vec![ - rpc_gas_price_wei - 1, - rpc_gas_price_wei, - rpc_gas_price_wei + 1, - rpc_gas_price_wei - 123_456, - rpc_gas_price_wei + 456_789, - ] - .into_iter() - .enumerate() - .map(|(idx, prev_gas_price_wei)| { - let account = make_payable_account((idx as u64 + 1) * 3_000); - RetryTxTemplate { - base: BaseTxTemplate::from(&account), - prev_gas_price_wei, - prev_nonce: idx as u64, - } - }) - .collect() + prev_gas_prices + .iter() + .copied() + .enumerate() + .map(|(idx, prev_gas_price_wei)| { + let account = make_payable_account((idx as u64 + 1) * 3_000); + RetryTxTemplate { + base: BaseTxTemplate::from(&account), + prev_gas_price_wei, + prev_nonce: idx as u64, + } + }) + .collect() }; let subject = BlockchainAgentWeb3::new( - rpc_gas_price_wei, + latest_gas_price_wei, 77_777, consuming_wallet, consuming_wallet_balances, @@ -645,15 +654,11 @@ mod tests { let result = subject.estimate_transaction_fee_total(&priced_qualified_payables); - let gas_prices_for_accounts_from_1_to_5 = vec![ - increase_gas_price_by_margin(rpc_gas_price_wei), - increase_gas_price_by_margin(rpc_gas_price_wei), - increase_gas_price_by_margin(rpc_gas_price_wei + 1), - increase_gas_price_by_margin(rpc_gas_price_wei), - increase_gas_price_by_margin(rpc_gas_price_wei + 456_789), - ]; - let expected_result = gas_prices_for_accounts_from_1_to_5 + let expected_result = prev_gas_prices .into_iter() + .map(|prev_gas_price_wei| { + PricedRetryTxTemplate::compute_gas_price(latest_gas_price_wei, prev_gas_price_wei) + }) .sum::() * (77_777 + WEB3_MAXIMAL_GAS_LIMIT_MARGIN); assert_eq!(result, expected_result) diff --git a/node/src/blockchain/blockchain_bridge.rs b/node/src/blockchain/blockchain_bridge.rs index 183f58659..0d13ead5d 100644 --- a/node/src/blockchain/blockchain_bridge.rs +++ b/node/src/blockchain/blockchain_bridge.rs @@ -39,7 +39,7 @@ use actix::{Addr, Recipient}; use futures::Future; use itertools::{Either, Itertools}; use masq_lib::blockchains::chains::Chain; -use masq_lib::constants::DEFAULT_GAS_PRICE_MARGIN; +use masq_lib::constants::DEFAULT_GAS_PRICE_RETRY_PERCENTAGE; use masq_lib::logger::Logger; use masq_lib::ui_gateway::NodeFromUiMessage; use regex::Regex; @@ -312,28 +312,32 @@ impl BlockchainBridge { Box::new( self.process_payments(msg.agent, msg.priced_templates) - .map_err(move |e: LocalPayableError| { - sent_payable_subs_err - .try_send(SentPayables { - payment_procedure_result: Self::payment_procedure_result_from_error( - e.clone(), - ), - payable_scan_type, - response_skeleton_opt: skeleton_opt, - }) - .expect("Accountant is dead"); - - format!("ReportAccountsPayable: {}", e) - }) - .and_then(move |batch_results| { - sent_payable_subs_success - .try_send(SentPayables { - payment_procedure_result: Ok(batch_results), - payable_scan_type, - response_skeleton_opt: skeleton_opt, - }) - .expect("Accountant is dead"); - + .then(move |result| { + match result { + Ok(batch_results) => { + sent_payable_subs_success + .try_send(SentPayables { + payment_procedure_result: Ok(batch_results), + payable_scan_type, + response_skeleton_opt: skeleton_opt, + }) + .expect("Accountant is dead"); + } + Err(e) => { + sent_payable_subs_err + .try_send(SentPayables { + payment_procedure_result: + Self::payment_procedure_result_from_error(e), + payable_scan_type, + response_skeleton_opt: skeleton_opt, + }) + .expect("Accountant is dead"); + } + } + // TODO Temporary workaround: prevents the Accountant from receiving two messages + // describing the same error. Duplicate notifications could previously trigger + // a panic in the scanners, because the substitution call for a given scanner + // was executed twice and tripped the guard that enforces a single concurrent scan. Ok(()) }), ) @@ -509,8 +513,8 @@ impl BlockchainBridge { pub fn extract_max_block_count(error: BlockchainInterfaceError) -> Option { let regex_result = - Regex::new(r".* (max: |allowed for your plan: |is limited to |block range limit \(|exceeds max block range )(?P\d+).*") - .expect("Invalid regex"); + Regex::new(r".* (max: |allowed for your plan: |is limited to |block range limit \(|exceeds max block range |maximum allowed is )(?P\d+).*") + .expect("Invalid regex"); let max_block_count = match error { BlockchainInterfaceError::QueryFailed(msg) => match regex_result.captures(msg.as_str()) { @@ -538,8 +542,8 @@ struct PendingTxInfo { when_sent: SystemTime, } -pub fn increase_gas_price_by_margin(gas_price: u128) -> u128 { - (gas_price * (100 + DEFAULT_GAS_PRICE_MARGIN as u128)) / 100 +pub fn increase_by_percentage(gas_price: u128) -> u128 { + (gas_price * (100 + DEFAULT_GAS_PRICE_RETRY_PERCENTAGE as u128)) / 100 } pub struct BlockchainBridgeSubsFactoryReal {} @@ -771,7 +775,7 @@ mod tests { let accountant_received_payment = accountant_recording_arc.lock().unwrap(); let blockchain_agent_with_context_msg_actual: &PricedTemplatesMessage = accountant_received_payment.get_record(0); - let computed_gas_price_wei = increase_gas_price_by_margin(0x230000000); + let computed_gas_price_wei = increase_by_percentage(0x230000000); let expected_tx_templates = tx_templates .iter() .map(|tx_template| PricedNewTxTemplate { @@ -1021,7 +1025,7 @@ mod tests { // let pending_payable_fingerprint_seeds_msg = // accountant_recording.get_record::(0); let sent_payables_msg = accountant_recording.get_record::(0); - let scan_error_msg = accountant_recording.get_record::(1); + // let scan_error_msg = accountant_recording.get_record::(1); let batch_results = sent_payables_msg.clone().payment_procedure_result.unwrap(); let failed_tx = FailedTx { hash: H256::from_str( @@ -1048,22 +1052,23 @@ mod tests { // amount: account.balance_wei // }] // ); - assert_eq!(scan_error_msg.scan_type, DetailedScanType::NewPayables); - assert_eq!( - scan_error_msg.response_skeleton_opt, - Some(ResponseSkeleton { - client_id: 1234, - context_id: 4321 - }) - ); - assert!(scan_error_msg - .msg - .contains("ReportAccountsPayable: Sending error: \"Transport error: Error(IncompleteMessage)\". Signed and hashed transactions:"), "This string didn't contain the expected: {}", scan_error_msg.msg); - assert!(scan_error_msg.msg.contains( - "FailedTx { hash: 0x81d20df32920161727cd20e375e53c2f9df40fd80256a236fb39e444c999fb6c," - )); - assert!(scan_error_msg.msg.contains("FailedTx { hash: 0x81d20df32920161727cd20e375e53c2f9df40fd80256a236fb39e444c999fb6c, receiver_address: 0x00000000000000000000000000000000626c6168, amount_minor: 111420204, timestamp:"), "This string didn't contain the expected: {}", scan_error_msg.msg); - assert_eq!(accountant_recording.len(), 2); + // assert_eq!(scan_error_msg.scan_type, DetailedScanType::NewPayables); + // assert_eq!( + // scan_error_msg.response_skeleton_opt, + // Some(ResponseSkeleton { + // client_id: 1234, + // context_id: 4321 + // }) + // ); + // assert!(scan_error_msg + // .msg + // .contains("ReportAccountsPayable: Sending error: \"Transport error: Error(IncompleteMessage)\". Signed and hashed transactions:"), "This string didn't contain the expected: {}", scan_error_msg.msg); + // assert!(scan_error_msg.msg.contains( + // "FailedTx { hash: 0x81d20df32920161727cd20e375e53c2f9df40fd80256a236fb39e444c999fb6c," + // )); + // assert!(scan_error_msg.msg.contains("FailedTx { hash: 0x81d20df32920161727cd20e375e53c2f9df40fd80256a236fb39e444c999fb6c, receiver_address: 0x00000000000000000000000000000000626c6168, amount_minor: 111420204, timestamp:"), "This string didn't contain the expected: {}", scan_error_msg.msg); + //assert_eq!(accountant_recording.len(), 2); + assert_eq!(accountant_recording.len(), 1); } #[test] @@ -1534,7 +1539,7 @@ mod tests { system.run(); let after = SystemTime::now(); let expected_transactions = RetrievedBlockchainTransactions { - new_start_block: BlockMarker::Value(42 + 9_000_000 + 1), + new_start_block: BlockMarker::Value(42 + 9_000_000), transactions: vec![ BlockchainTransaction { block_number: 6040059, @@ -1735,7 +1740,7 @@ mod tests { let received_payments_message = accountant_recording.get_record::(0); check_timestamp(before, received_payments_message.timestamp, after); let expected_transactions = RetrievedBlockchainTransactions { - new_start_block: BlockMarker::Value(6 + 5000 + 1), + new_start_block: BlockMarker::Value(6 + 5000), transactions: vec![BlockchainTransaction { block_number: 2000, from: earning_wallet.clone(), @@ -2198,6 +2203,15 @@ mod tests { assert_eq!(Some(100000), max_block_count); } + #[test] + fn extract_max_block_range_for_nodies_error_response_v2() { + let result = BlockchainInterfaceError::QueryFailed("RPC error: Error { code: ServerError(-32001), message: \"Block range too large: maximum allowed is 20000 blocks\", data: None }".to_string()); + + let max_block_count = BlockchainBridge::extract_max_block_count(result); + + assert_eq!(Some(20000), max_block_count); + } + #[test] fn extract_max_block_range_for_expected_batch_got_single_error_response() { let result = BlockchainInterfaceError::QueryFailed( @@ -2225,7 +2239,7 @@ mod tests { #[test] fn increase_gas_price_by_margin_works() { - assert_eq!(increase_gas_price_by_margin(1_000_000_000), 1_300_000_000); - assert_eq!(increase_gas_price_by_margin(9_000_000_000), 11_700_000_000); + assert_eq!(increase_by_percentage(1_000_000_000), 1_300_000_000); + assert_eq!(increase_by_percentage(9_000_000_000), 11_700_000_000); } } diff --git a/node/src/blockchain/blockchain_interface/blockchain_interface_web3/lower_level_interface_web3.rs b/node/src/blockchain/blockchain_interface/blockchain_interface_web3/lower_level_interface_web3.rs index 7a4d6ddfb..e2c6f4dcc 100644 --- a/node/src/blockchain/blockchain_interface/blockchain_interface_web3/lower_level_interface_web3.rs +++ b/node/src/blockchain/blockchain_interface/blockchain_interface_web3/lower_level_interface_web3.rs @@ -9,7 +9,7 @@ use futures::Future; use serde_json::Value; use web3::contract::{Contract, Options}; use web3::transports::{Batch, Http}; -use web3::types::{Address, BlockNumber, Filter, Log}; +use web3::types::{Address, Filter, Log}; use web3::{Error, Web3}; pub struct LowBlockchainIntWeb3 { @@ -68,7 +68,7 @@ impl LowBlockchainInt for LowBlockchainIntWeb3 { Box::new( self.web3 .eth() - .transaction_count(address, Some(BlockNumber::Pending)) + .transaction_count(address, None) .map_err(move |e| QueryFailed(format!("{} for wallet {}", e, address))), ) } diff --git a/node/src/blockchain/blockchain_interface/blockchain_interface_web3/mod.rs b/node/src/blockchain/blockchain_interface/blockchain_interface_web3/mod.rs index 9249c6ee0..f7e779cc3 100644 --- a/node/src/blockchain/blockchain_interface/blockchain_interface_web3/mod.rs +++ b/node/src/blockchain/blockchain_interface/blockchain_interface_web3/mod.rs @@ -279,6 +279,8 @@ impl BlockchainInterface for BlockchainInterfaceWeb3 { get_transaction_id .map_err(LocalPayableError::TransactionID) .and_then(move |latest_nonce| { + debug!(logger, "Latest nonce: {:?}", latest_nonce); + let templates = SignableTxTemplates::new(priced_templates, latest_nonce.as_u64()); @@ -394,7 +396,9 @@ impl BlockchainInterfaceWeb3 { ) -> BlockMarker { let locally_determined_end_block_marker = match (start_block_marker, scan_range) { (BlockMarker::Value(start_block), BlockScanRange::Range(scan_range_number)) => { - BlockMarker::Value(start_block + scan_range_number) + // Subtract 1 because the range is inclusive: [start_block, end_block] + // Example: If max range is 20000, we need start_block to start_block+20000-1 (ending up with 20000 blocks total) + BlockMarker::Value(start_block + scan_range_number - 1) } (_, _) => BlockMarker::Uninitialized, }; @@ -469,7 +473,6 @@ mod tests { use super::*; use crate::accountant::scanners::pending_payable_scanner::utils::TxHashByTable; use crate::accountant::test_utils::make_payable_account; - use crate::blockchain::blockchain_bridge::increase_gas_price_by_margin; use crate::blockchain::blockchain_interface::blockchain_interface_web3::{ BlockchainInterfaceWeb3, CONTRACT_ABI, REQUESTS_IN_PARALLEL, TRANSACTION_LITERAL, TRANSFER_METHOD_ID, @@ -501,10 +504,12 @@ mod tests { use itertools::Either; use web3::transports::Http; use web3::types::{H256, U256}; + use masq_lib::constants::DEFAULT_GAS_PRICE_RETRY_CONSTANT; use crate::accountant::scanners::payable_scanner::tx_templates::initial::new::NewTxTemplates; use crate::accountant::scanners::payable_scanner::tx_templates::initial::retry::RetryTxTemplates; use crate::accountant::scanners::payable_scanner::tx_templates::priced::retry::PricedRetryTxTemplate; use crate::accountant::scanners::payable_scanner::tx_templates::test_utils::RetryTxTemplateBuilder; + use crate::blockchain::blockchain_bridge::increase_by_percentage; #[test] fn constants_are_correct() { @@ -560,8 +565,8 @@ mod tests { let start_block_marker = BlockMarker::Value(42); let scan_range = BlockScanRange::Range(1000); let block_response = "0x7d0"; // 2_000 - let expected_new_start_block = BlockMarker::Value(42 + 1000 + 1); - let expected_log = "from start block: Number(42) to end block: Number(1042)"; + let expected_new_start_block = BlockMarker::Value(42 + 1000); + let expected_log = "from start block: Number(42) to end block: Number(1041)"; assert_on_retrieves_transactions( start_block_marker, scan_range, @@ -884,7 +889,7 @@ mod tests { let gas_price_wei_from_rpc_u128_wei = u128::from_str_radix(&gas_price_wei_from_rpc_hex[2..], 16).unwrap(); let gas_price_wei_from_rpc_u128_wei_with_margin = - increase_gas_price_by_margin(gas_price_wei_from_rpc_u128_wei); + increase_by_percentage(gas_price_wei_from_rpc_u128_wei); let expected_result = Either::Left(PricedNewTxTemplates::new( tx_templates.clone(), gas_price_wei_from_rpc_u128_wei_with_margin, @@ -902,32 +907,32 @@ mod tests { #[test] fn blockchain_interface_web3_can_introduce_blockchain_agent_in_the_retry_payables_mode() { let gas_price_wei = "0x3B9ACA00"; // 1000000000 - let gas_price_from_rpc = u128::from_str_radix(&gas_price_wei[2..], 16).unwrap(); + let latest_gas_price_wei = u128::from_str_radix(&gas_price_wei[2..], 16).unwrap(); let retry_1 = RetryTxTemplateBuilder::default() .payable_account(&make_payable_account(12)) - .prev_gas_price_wei(gas_price_from_rpc - 1) + .prev_gas_price_wei(latest_gas_price_wei - 1) .build(); let retry_2 = RetryTxTemplateBuilder::default() .payable_account(&make_payable_account(34)) - .prev_gas_price_wei(gas_price_from_rpc) + .prev_gas_price_wei(latest_gas_price_wei) .build(); let retry_3 = RetryTxTemplateBuilder::default() .payable_account(&make_payable_account(56)) - .prev_gas_price_wei(gas_price_from_rpc + 1) + .prev_gas_price_wei(latest_gas_price_wei + 1) .build(); let retry_tx_templates = RetryTxTemplates(vec![retry_1.clone(), retry_2.clone(), retry_3.clone()]); let expected_retry_tx_templates = PricedRetryTxTemplates(vec![ - PricedRetryTxTemplate::new(retry_1, increase_gas_price_by_margin(gas_price_from_rpc)), - PricedRetryTxTemplate::new(retry_2, increase_gas_price_by_margin(gas_price_from_rpc)), + PricedRetryTxTemplate::new(retry_1, increase_by_percentage(latest_gas_price_wei)), + PricedRetryTxTemplate::new(retry_2, increase_by_percentage(latest_gas_price_wei)), PricedRetryTxTemplate::new( retry_3, - increase_gas_price_by_margin(gas_price_from_rpc + 1), + (latest_gas_price_wei + 1) + DEFAULT_GAS_PRICE_RETRY_CONSTANT, ), ]); - let expected_estimated_transaction_fee_total = 285_979_200_073_328; + let expected_estimated_transaction_fee_total = 263_981_166_713_328; test_blockchain_interface_web3_can_introduce_blockchain_agent( Either::Right(retry_tx_templates), @@ -1270,7 +1275,7 @@ mod tests { Err(BlockchainInterfaceError::InvalidResponse), &logger ), - BlockMarker::Value(150) + BlockMarker::Value(149) ); assert_eq!( Subject::calculate_end_block_marker( @@ -1288,7 +1293,7 @@ mod tests { Ok(120.into()), &logger ), - BlockMarker::Value(50 + 10) + BlockMarker::Value(59) ); } diff --git a/node/src/blockchain/blockchain_interface/blockchain_interface_web3/utils.rs b/node/src/blockchain/blockchain_interface/blockchain_interface_web3/utils.rs index 16cab8c2d..fa893b19f 100644 --- a/node/src/blockchain/blockchain_interface/blockchain_interface_web3/utils.rs +++ b/node/src/blockchain/blockchain_interface/blockchain_interface_web3/utils.rs @@ -282,11 +282,16 @@ pub fn send_payables_within_batch( transmission_log(chain, &signable_tx_templates) ); + let logger_clone = logger.clone(); + Box::new( web3_batch .transport() .submit_batch() - .map_err(move |e| return_sending_error(&sent_txs_for_err, &e)) + .map_err(move |e| { + warning!(logger_clone, "Failed to submit batch to Web3 client: {}", e); + return_sending_error(&sent_txs_for_err, &e) + }) .and_then(move |batch_responses| Ok(return_batch_results(sent_txs, batch_responses))), ) } @@ -727,6 +732,7 @@ mod tests { #[test] fn send_payables_within_batch_fails_on_submit_batch_call() { + let test_name = "send_payables_within_batch_fails_on_submit_batch_call"; let port = find_free_port(); let (_event_loop_handle, transport) = Http::with_max_parallel( &format!("http://{}:{}", &Ipv4Addr::LOCALHOST.to_string(), port), @@ -779,12 +785,15 @@ mod tests { failed_txs, }); - test_send_payables_within_batch( - "send_payables_within_batch_fails_on_submit_batch_call", - signable_tx_templates, - expected_result, - port, - ); + test_send_payables_within_batch(test_name, signable_tx_templates, expected_result, port); + + let os_specific_code = transport_error_code(); + let os_specific_msg = transport_error_message(); + TestLogHandler::new().exists_log_containing(&format!( + "WARN: {test_name}: Failed to submit batch to Web3 client: Transport error: \ + Error(Connect, Os {{ code: {}, kind: ConnectionRefused, message: \"{}\" }}", + os_specific_code, os_specific_msg + )); } #[test] diff --git a/node/src/blockchain/blockchain_interface_initializer.rs b/node/src/blockchain/blockchain_interface_initializer.rs index 04838f312..01b97a3b8 100644 --- a/node/src/blockchain/blockchain_interface_initializer.rs +++ b/node/src/blockchain/blockchain_interface_initializer.rs @@ -47,7 +47,7 @@ mod tests { use crate::accountant::scanners::payable_scanner::tx_templates::initial::new::NewTxTemplates; use crate::accountant::scanners::payable_scanner::tx_templates::priced::new::PricedNewTxTemplates; use crate::accountant::test_utils::make_payable_account; - use crate::blockchain::blockchain_bridge::increase_gas_price_by_margin; + use crate::blockchain::blockchain_bridge::increase_by_percentage; use crate::blockchain::blockchain_interface_initializer::BlockchainInterfaceInitializer; use crate::test_utils::make_wallet; use futures::Future; @@ -88,7 +88,7 @@ mod tests { .unwrap(); assert_eq!(blockchain_agent.consuming_wallet(), &payable_wallet); let result = blockchain_agent.price_qualified_payables(Either::Left(tx_templates.clone())); - let gas_price_with_margin = increase_gas_price_by_margin(1_000_000_000); + let gas_price_with_margin = increase_by_percentage(1_000_000_000); let expected_result = Either::Left(PricedNewTxTemplates::new( tx_templates, gas_price_with_margin, diff --git a/node/src/bootstrapper.rs b/node/src/bootstrapper.rs index d0b92755c..7c3891a8a 100644 --- a/node/src/bootstrapper.rs +++ b/node/src/bootstrapper.rs @@ -361,6 +361,7 @@ pub struct BootstrapperConfig { pub node_descriptor: NodeDescriptor, pub cryptde_pair: CryptDEPair, pub mapping_protocol_opt: Option, + pub new_public_key_opt: Option, pub real_user: RealUser, pub payment_thresholds_opt: Option, @@ -406,6 +407,7 @@ impl BootstrapperConfig { Box::new(CryptDEReal::disabled()), ), mapping_protocol_opt: None, + new_public_key_opt: None, real_user: RealUser::new(None, None, None), payment_thresholds_opt: Default::default(), @@ -748,7 +750,7 @@ mod tests { use tokio::prelude::Async; lazy_static! { - static ref CRYPTDE_PAIR: CryptDEPair = CryptDEPair::null(); + static ref BS_CRYPTDE_PAIR: CryptDEPair = CryptDEPair::null(); pub static ref INITIALIZATION: Mutex = Mutex::new(false); } @@ -1535,13 +1537,13 @@ mod tests { ); let cryptde_ref = { let descriptor = Bootstrapper::make_local_descriptor( - CRYPTDE_PAIR.main.as_ref(), + BS_CRYPTDE_PAIR.main.as_ref(), Some(node_addr), TEST_DEFAULT_CHAIN, ); - Bootstrapper::report_local_descriptor(CRYPTDE_PAIR.main.as_ref(), &descriptor); + Bootstrapper::report_local_descriptor(BS_CRYPTDE_PAIR.main.as_ref(), &descriptor); - CRYPTDE_PAIR.main.as_ref() + BS_CRYPTDE_PAIR.main.as_ref() }; let expected_descriptor = format!( "masq://base-sepolia:{}@2.3.4.5:3456/4567", @@ -1577,13 +1579,13 @@ mod tests { init_test_logging(); let cryptdes = { let descriptor = Bootstrapper::make_local_descriptor( - CRYPTDE_PAIR.main.as_ref(), + BS_CRYPTDE_PAIR.main.as_ref(), None, TEST_DEFAULT_CHAIN, ); - Bootstrapper::report_local_descriptor(CRYPTDE_PAIR.main.as_ref(), &descriptor); + Bootstrapper::report_local_descriptor(BS_CRYPTDE_PAIR.main.as_ref(), &descriptor); - CRYPTDE_PAIR.clone() + BS_CRYPTDE_PAIR.clone() }; let expected_descriptor = format!( "masq://base-sepolia:{}@:", diff --git a/node/src/daemon/setup_reporter.rs b/node/src/daemon/setup_reporter.rs index dd1e2182a..f0d31f2be 100644 --- a/node/src/daemon/setup_reporter.rs +++ b/node/src/daemon/setup_reporter.rs @@ -1041,6 +1041,26 @@ impl ValueRetriever for Neighbors { } } +struct NewPublicKey {} +impl ValueRetriever for NewPublicKey { + fn value_name(&self) -> &'static str { + "new-public-key" + } + + fn computed_default( + &self, + _bootstrapper_config: &BootstrapperConfig, + _persistent_config: &dyn PersistentConfiguration, + _db_password_opt: &Option, + ) -> Option<(String, UiSetupResponseValueStatus)> { + Some(("".to_string(), Blank)) + } + + fn is_required(&self, _params: &SetupCluster) -> bool { + false + } +} + struct PaymentThresholds {} impl ValueRetriever for PaymentThresholds { fn value_name(&self) -> &'static str { @@ -1216,6 +1236,7 @@ fn value_retrievers(dirs_wrapper: &dyn DirsWrapper) -> Vec, ) -> rusqlite::Result<()> { - let sql_statement_for_sent_payable = "create table if not exists sent_payable ( - rowid integer primary key, - tx_hash text not null, - receiver_address text not null, - amount_high_b integer not null, - amount_low_b integer not null, - timestamp integer not null, - gas_price_wei_high_b integer not null, - gas_price_wei_low_b integer not null, - nonce integer not null, - status text not null - )"; - - let sql_statement_for_failed_payable = "create table if not exists failed_payable ( - rowid integer primary key, - tx_hash text not null, - receiver_address text not null, - amount_high_b integer not null, - amount_low_b integer not null, - timestamp integer not null, - gas_price_wei_high_b integer not null, - gas_price_wei_low_b integer not null, - nonce integer not null, - reason text not null, - status text not null - )"; - - let sql_statement_for_pending_payable = "drop table pending_payable"; - - declaration_utils.execute_upon_transaction(&[ - &sql_statement_for_sent_payable, - &sql_statement_for_failed_payable, - &sql_statement_for_pending_payable, - ]) + declaration_utils.execute_upon_transaction(&[&format!( + "INSERT INTO config (name, value, encrypted) VALUES ('rate_pack_limits', '{}', 0)", + DEFAULT_RATE_PACK_LIMITS.rate_pack_limits_parameter() + )]) } fn old_version(&self) -> usize { @@ -56,17 +28,14 @@ mod tests { use crate::database::db_initializer::{ DbInitializationConfig, DbInitializer, DbInitializerReal, DATABASE_FILE, }; - use crate::database::test_utils::{ - SQL_ATTRIBUTES_FOR_CREATING_FAILED_PAYABLE, SQL_ATTRIBUTES_FOR_CREATING_SENT_PAYABLE, + use crate::test_utils::database_utils::{ + bring_db_0_back_to_life_and_return_connection, make_external_data, retrieve_config_row, }; - use crate::test_utils::database_utils::{assert_create_table_stm_contains_all_parts, assert_table_does_not_exist, assert_table_exists, bring_db_0_back_to_life_and_return_connection, make_external_data}; - use masq_lib::test_utils::logging::{init_test_logging, TestLogHandler}; use masq_lib::test_utils::utils::ensure_node_home_directory_exists; use std::fs::create_dir_all; #[test] - fn migration_from_11_to_12_is_applied_correctly() { - init_test_logging(); + fn migration_from_11_to_12_is_properly_set() { let dir_path = ensure_node_home_directory_exists( "db_migrations", "migration_from_11_to_12_is_properly_set", @@ -75,37 +44,34 @@ mod tests { let db_path = dir_path.join(DATABASE_FILE); let _ = bring_db_0_back_to_life_and_return_connection(&db_path); let subject = DbInitializerReal::default(); - let _prev_connection = subject - .initialize_to_version( - &dir_path, - 11, - DbInitializationConfig::create_or_migrate(make_external_data()), - ) - .unwrap(); - let connection = subject - .initialize_to_version( - &dir_path, - 12, - DbInitializationConfig::create_or_migrate(make_external_data()), - ) - .unwrap(); + let result = subject.initialize_to_version( + &dir_path, + 11, + DbInitializationConfig::create_or_migrate(make_external_data()), + ); + + assert!(result.is_ok()); - assert_table_exists(connection.as_ref(), "sent_payable"); - assert_table_exists(connection.as_ref(), "failed_payable"); - assert_create_table_stm_contains_all_parts( - &*connection, - "sent_payable", - SQL_ATTRIBUTES_FOR_CREATING_SENT_PAYABLE, + let result = subject.initialize_to_version( + &dir_path, + 12, + DbInitializationConfig::create_or_migrate(make_external_data()), ); - assert_create_table_stm_contains_all_parts( - &*connection, - "failed_payable", - SQL_ATTRIBUTES_FOR_CREATING_FAILED_PAYABLE, + + assert!(result.is_ok()); + let connection = result.unwrap(); + let (lc_value, lc_encrypted) = retrieve_config_row(connection.as_ref(), "rate_pack_limits"); + let (cs_value, cs_encrypted) = retrieve_config_row(connection.as_ref(), "schema_version"); + assert_eq!( + lc_value, + Some( + "100-100000000000000|100-100000000000000|100-100000000000000|100-100000000000000" + .to_string() + ) ); - assert_table_does_not_exist(connection.as_ref(), "pending_payable"); - TestLogHandler::new().assert_logs_contain_in_order(vec![ - "DbMigrator: Database successfully migrated from version 11 to 12", - ]); + assert_eq!(lc_encrypted, false); + assert_eq!(cs_value, Some("12".to_string())); + assert_eq!(cs_encrypted, false) } } diff --git a/node/src/database/db_migrations/migrations/migration_12_to_13.rs b/node/src/database/db_migrations/migrations/migration_12_to_13.rs new file mode 100644 index 000000000..3a83cb11c --- /dev/null +++ b/node/src/database/db_migrations/migrations/migration_12_to_13.rs @@ -0,0 +1,111 @@ +// Copyright (c) 2025, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. +use crate::database::db_migrations::db_migrator::DatabaseMigration; +use crate::database::db_migrations::migrator_utils::DBMigDeclarator; + +#[allow(non_camel_case_types)] +pub struct Migrate_12_to_13; + +impl DatabaseMigration for Migrate_12_to_13 { + fn migrate<'a>( + &self, + declaration_utils: Box, + ) -> rusqlite::Result<()> { + let sql_statement_for_sent_payable = "create table if not exists sent_payable ( + rowid integer primary key, + tx_hash text not null, + receiver_address text not null, + amount_high_b integer not null, + amount_low_b integer not null, + timestamp integer not null, + gas_price_wei_high_b integer not null, + gas_price_wei_low_b integer not null, + nonce integer not null, + status text not null + )"; + + let sql_statement_for_failed_payable = "create table if not exists failed_payable ( + rowid integer primary key, + tx_hash text not null, + receiver_address text not null, + amount_high_b integer not null, + amount_low_b integer not null, + timestamp integer not null, + gas_price_wei_high_b integer not null, + gas_price_wei_low_b integer not null, + nonce integer not null, + reason text not null, + status text not null + )"; + + let sql_statement_for_pending_payable = "drop table pending_payable"; + + declaration_utils.execute_upon_transaction(&[ + &sql_statement_for_sent_payable, + &sql_statement_for_failed_payable, + &sql_statement_for_pending_payable, + ]) + } + + fn old_version(&self) -> usize { + 12 + } +} + +#[cfg(test)] +mod tests { + use crate::database::db_initializer::{ + DbInitializationConfig, DbInitializer, DbInitializerReal, DATABASE_FILE, + }; + use crate::database::test_utils::{ + SQL_ATTRIBUTES_FOR_CREATING_FAILED_PAYABLE, SQL_ATTRIBUTES_FOR_CREATING_SENT_PAYABLE, + }; + use crate::test_utils::database_utils::{assert_create_table_stm_contains_all_parts, assert_table_does_not_exist, assert_table_exists, bring_db_0_back_to_life_and_return_connection, make_external_data}; + use masq_lib::test_utils::logging::{init_test_logging, TestLogHandler}; + use masq_lib::test_utils::utils::ensure_node_home_directory_exists; + use std::fs::create_dir_all; + + #[test] + fn migration_from_12_to_13_is_applied_correctly() { + init_test_logging(); + let dir_path = ensure_node_home_directory_exists( + "db_migrations", + "migration_from_12_to_13_is_applied_correctly", + ); + create_dir_all(&dir_path).unwrap(); + let db_path = dir_path.join(DATABASE_FILE); + let _ = bring_db_0_back_to_life_and_return_connection(&db_path); + let subject = DbInitializerReal::default(); + let _prev_connection = subject + .initialize_to_version( + &dir_path, + 12, + DbInitializationConfig::create_or_migrate(make_external_data()), + ) + .unwrap(); + + let connection = subject + .initialize_to_version( + &dir_path, + 13, + DbInitializationConfig::create_or_migrate(make_external_data()), + ) + .unwrap(); + + assert_table_exists(connection.as_ref(), "sent_payable"); + assert_table_exists(connection.as_ref(), "failed_payable"); + assert_create_table_stm_contains_all_parts( + &*connection, + "sent_payable", + SQL_ATTRIBUTES_FOR_CREATING_SENT_PAYABLE, + ); + assert_create_table_stm_contains_all_parts( + &*connection, + "failed_payable", + SQL_ATTRIBUTES_FOR_CREATING_FAILED_PAYABLE, + ); + assert_table_does_not_exist(connection.as_ref(), "pending_payable"); + TestLogHandler::new().assert_logs_contain_in_order(vec![ + "DbMigrator: Database successfully migrated from version 12 to 13", + ]); + } +} diff --git a/node/src/database/db_migrations/migrations/mod.rs b/node/src/database/db_migrations/migrations/mod.rs index 1abcf911c..44d4dac82 100644 --- a/node/src/database/db_migrations/migrations/mod.rs +++ b/node/src/database/db_migrations/migrations/mod.rs @@ -14,3 +14,5 @@ pub mod migration_9_to_10; pub mod migration_10_to_11; #[rustfmt::skip] pub mod migration_11_to_12; +#[rustfmt::skip] +pub mod migration_12_to_13; diff --git a/node/src/db_config/config_dao_null.rs b/node/src/db_config/config_dao_null.rs index 1c988caea..8ee6716d2 100644 --- a/node/src/db_config/config_dao_null.rs +++ b/node/src/db_config/config_dao_null.rs @@ -6,7 +6,7 @@ use crate::db_config::config_dao::{ConfigDao, ConfigDaoError, ConfigDaoRecord}; use crate::neighborhood::DEFAULT_MIN_HOPS; use crate::sub_lib::accountant; use crate::sub_lib::accountant::DEFAULT_PAYMENT_THRESHOLDS; -use crate::sub_lib::neighborhood::DEFAULT_RATE_PACK; +use crate::sub_lib::neighborhood::{DEFAULT_RATE_PACK, DEFAULT_RATE_PACK_LIMITS}; use itertools::Itertools; use masq_lib::blockchains::chains::Chain; use masq_lib::constants::{CURRENT_SCHEMA_VERSION, DEFAULT_GAS_PRICE}; @@ -138,6 +138,13 @@ impl Default for ConfigDaoNull { "rate_pack".to_string(), (Some(DEFAULT_RATE_PACK.to_string()), false), ); + data.insert( + "rate_pack_limits".to_string(), + ( + Some(DEFAULT_RATE_PACK_LIMITS.rate_pack_limits_parameter()), + false, + ), + ); data.insert( "scan_intervals".to_string(), ( diff --git a/node/src/db_config/persistent_configuration.rs b/node/src/db_config/persistent_configuration.rs index 8567d7807..602593ab7 100644 --- a/node/src/db_config/persistent_configuration.rs +++ b/node/src/db_config/persistent_configuration.rs @@ -3,6 +3,7 @@ use crate::arbitrary_id_stamp_in_trait; use crate::blockchain::bip32::Bip32EncryptionKeyProvider; use crate::blockchain::bip39::{Bip39, Bip39Error}; +use crate::database::db_initializer::{DbInitializationConfig, DbInitializer, DbInitializerReal}; use crate::database::rusqlite_wrappers::{ConnectionWrapper, TransactionSafeWrapper}; use crate::db_config::config_dao::{ConfigDao, ConfigDaoError, ConfigDaoReal, ConfigDaoRecord}; use crate::db_config::secure_config_layer::{SecureConfigLayer, SecureConfigLayerError}; @@ -14,19 +15,28 @@ use crate::sub_lib::accountant::{PaymentThresholds, ScanIntervals}; use crate::sub_lib::cryptde::{CryptDE, PlainData}; use crate::sub_lib::cryptde_null::CryptDENull; use crate::sub_lib::cryptde_real::CryptDEReal; -use crate::sub_lib::neighborhood::{Hops, NodeDescriptor, RatePack}; +use crate::sub_lib::neighborhood::{Hops, NodeDescriptor, RatePack, RatePackLimits}; +use crate::sub_lib::utils::db_connection_launch_panic; use crate::sub_lib::wallet::Wallet; +use lazy_static::lazy_static; use masq_lib::blockchains::chains::Chain; use masq_lib::constants::{HIGHEST_USABLE_PORT, LOWEST_USABLE_INSECURE_PORT}; use masq_lib::shared_schema::{ConfiguratorError, ParamError}; use masq_lib::utils::NeighborhoodModeLight; use masq_lib::utils::{to_string, AutomapProtocol}; +use regex::{Captures, Regex}; use rustc_hex::{FromHex, ToHex}; use std::fmt::Display; use std::net::{Ipv4Addr, SocketAddrV4, TcpListener}; +use std::path::PathBuf; use std::str::FromStr; use websocket::url::Url; +lazy_static! { + static ref RATE_PACK_LIMIT_FORMAT: Regex = + Regex::new(r"^(\d{1,19})-(\d{1,19})\|(\d{1,19})-(\d{1,19})\|(\d{1,19})-(\d{1,19})\|(\d{1,19})-(\d{1,19})$").unwrap(); +} + #[derive(Clone, PartialEq, Eq, Debug)] pub enum PersistentConfigError { NotPresent, @@ -159,6 +169,7 @@ pub trait PersistentConfiguration { fn payment_thresholds(&self) -> Result; fn set_payment_thresholds(&mut self, curves: String) -> Result<(), PersistentConfigError>; fn rate_pack(&self) -> Result; + fn rate_pack_limits(&self) -> Result; fn set_rate_pack(&mut self, rate_pack: String) -> Result<(), PersistentConfigError>; fn scan_intervals(&self) -> Result; fn set_scan_intervals(&mut self, intervals: String) -> Result<(), PersistentConfigError>; @@ -570,6 +581,44 @@ impl PersistentConfiguration for PersistentConfigurationReal { self.simple_set_method("rate_pack", rate_pack) } + fn rate_pack_limits(&self) -> Result { + let limits_string = self + .get("rate_pack_limits") + .expect( + "Required value rate_pack_limits missing from CONFIG table: database is corrupt!", + ) + .expect( + "Required value rate_pack_limits is NULL in CONFIG table: database is corrupt!", + ); + let captures = RATE_PACK_LIMIT_FORMAT.captures(limits_string.as_str()) + .unwrap_or_else(|| panic!("Syntax error in rate_pack_limits value '{}': should be -|-|-|- where L is low, H is high, R is routing, E is exit, BR is byte rate, and SR is service rate. All numbers should be in wei.", limits_string)); + let candidate = RatePackLimits::new( + Self::extract_candidate(&captures, 1), + Self::extract_candidate(&captures, 2), + ); + Self::check_rate_pack_limit_order( + candidate.lo.routing_byte_rate, + candidate.hi.routing_byte_rate, + "routing_byte_rate", + ); + Self::check_rate_pack_limit_order( + candidate.lo.routing_service_rate, + candidate.hi.routing_service_rate, + "routing_service_rate", + ); + Self::check_rate_pack_limit_order( + candidate.lo.exit_byte_rate, + candidate.hi.exit_byte_rate, + "exit_byte_rate", + ); + Self::check_rate_pack_limit_order( + candidate.lo.exit_service_rate, + candidate.hi.exit_service_rate, + "exit_service_rate", + ); + Ok(candidate) + } + fn scan_intervals(&self) -> Result { self.combined_params_get_method(|str: &str| ScanIntervals::try_from(str), "scan_intervals") } @@ -671,6 +720,239 @@ impl PersistentConfigurationReal { parameter_name ) } + + fn extract_candidate(captures: &Captures, start_index: usize) -> RatePack { + RatePack { + routing_byte_rate: Self::parse_capture(captures, start_index), + routing_service_rate: Self::parse_capture(captures, start_index + 2), + exit_byte_rate: Self::parse_capture(captures, start_index + 4), + exit_service_rate: Self::parse_capture(captures, start_index + 6), + } + } + + fn parse_capture(captures: &Captures, index: usize) -> u64 { + u64::from_str( + captures + .get(index) + .expect("Internal error: regex needs eight captures") + .as_str(), + ) + .expect("Internal error: regex must require u64") + } + + fn check_rate_pack_limit_order(low: u64, high: u64, field_name: &str) { + if low >= high { + panic!( + "Rate pack limits should have low limits less than high limits, but {} limits are {}-{}", + field_name, low, high + ); + } + } +} + +pub struct PersistentConfigurationInvalid {} + +impl PersistentConfiguration for PersistentConfigurationInvalid { + fn blockchain_service_url(&self) -> Result, PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn set_blockchain_service_url(&mut self, _url: &str) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn current_schema_version(&self) -> String { + PersistentConfigurationInvalid::invalid() + } + fn chain_name(&self) -> String { + PersistentConfigurationInvalid::invalid() + } + fn check_password( + &self, + _db_password_opt: Option, + ) -> Result { + PersistentConfigurationInvalid::invalid() + } + fn change_password( + &mut self, + _old_password_opt: Option, + _new_password: &str, + ) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn consuming_wallet( + &self, + _db_password: &str, + ) -> Result, PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn consuming_wallet_private_key( + &self, + _db_password: &str, + ) -> Result, PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn clandestine_port(&self) -> Result { + PersistentConfigurationInvalid::invalid() + } + fn set_clandestine_port(&mut self, _port: u16) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn cryptde( + &self, + _db_password: &str, + ) -> Result>, PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn set_cryptde( + &mut self, + _cryptde: &dyn CryptDE, + _db_password: &str, + ) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn earning_wallet(&self) -> Result, PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn earning_wallet_address(&self) -> Result, PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn gas_price(&self) -> Result { + PersistentConfigurationInvalid::invalid() + } + fn set_gas_price(&mut self, _gas_price: u64) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn mapping_protocol(&self) -> Result, PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn set_mapping_protocol( + &mut self, + _value_opt: Option, + ) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn min_hops(&self) -> Result { + PersistentConfigurationInvalid::invalid() + } + fn set_min_hops(&mut self, _value: Hops) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn neighborhood_mode(&self) -> Result { + PersistentConfigurationInvalid::invalid() + } + fn set_neighborhood_mode( + &mut self, + _value: NeighborhoodModeLight, + ) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn past_neighbors( + &self, + _db_password: &str, + ) -> Result>, PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn set_past_neighbors( + &mut self, + _node_descriptors_opt: Option>, + _db_password: &str, + ) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn start_block(&self) -> Result, PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn set_start_block(&mut self, _value_opt: Option) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn max_block_count(&self) -> Result, PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn set_max_block_count( + &mut self, + _value_opt: Option, + ) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn set_start_block_from_txn( + &mut self, + _value_opt: Option, + _transaction: &mut TransactionSafeWrapper, + ) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn set_wallet_info( + &mut self, + _consuming_wallet_private_key: &str, + _earning_wallet_address: &str, + _db_password: &str, + ) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn payment_thresholds(&self) -> Result { + PersistentConfigurationInvalid::invalid() + } + fn set_payment_thresholds(&mut self, _curves: String) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn rate_pack(&self) -> Result { + PersistentConfigurationInvalid::invalid() + } + fn set_rate_pack(&mut self, _rate_pack: String) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + fn rate_pack_limits(&self) -> Result { + PersistentConfigurationInvalid::invalid() + } + fn scan_intervals(&self) -> Result { + PersistentConfigurationInvalid::invalid() + } + fn set_scan_intervals(&mut self, _intervals: String) -> Result<(), PersistentConfigError> { + PersistentConfigurationInvalid::invalid() + } + arbitrary_id_stamp_in_trait!(); +} + +impl PersistentConfigurationInvalid { + pub fn new() -> Self { + Self {} + } + + fn invalid() -> ! { + panic!("PersistentConfiguration is uninitialized") + } +} + +impl Default for PersistentConfigurationInvalid { + fn default() -> Self { + Self::new() + } +} + +pub trait PersistentConfigurationFactory { + fn make(&self) -> Box; +} + +pub struct PersistentConfigurationFactoryReal { + data_directory: PathBuf, +} + +impl PersistentConfigurationFactory for PersistentConfigurationFactoryReal { + fn make(&self) -> Box { + let db_initializer: &dyn DbInitializer = &DbInitializerReal::default(); + let conn = db_initializer + .initialize( + self.data_directory.as_path(), + DbInitializationConfig::panic_on_migration(), + ) + .unwrap_or_else(|err| db_connection_launch_panic(err, &self.data_directory)); + Box::new(PersistentConfigurationReal::from(conn)) + } +} + +impl PersistentConfigurationFactoryReal { + pub fn new(data_directory: PathBuf) -> Self { + Self { data_directory } + } } #[cfg(test)] @@ -2284,6 +2566,166 @@ mod tests { getter_method_plain_data_does_not_tolerate_none_value!("rate_pack"); } + #[test] + fn rate_pack_limits_works() { + persistent_config_plain_data_assertions_for_simple_get_method!( + "rate_pack_limits", + "100-200|300-400|500000000000000000-600000000000000000|700-800", + RatePackLimits::new( + RatePack { + routing_byte_rate: 100, + routing_service_rate: 300, + exit_byte_rate: 500_000_000_000_000_000, + exit_service_rate: 700, + }, + RatePack { + routing_byte_rate: 200, + routing_service_rate: 400, + exit_byte_rate: 600_000_000_000_000_000, + exit_service_rate: 800, + } + ) + ); + } + + #[test] + #[should_panic( + expected = "Required value rate_pack_limits is NULL in CONFIG table: database is corrupt!" + )] + fn rate_pack_limits_panics_at_none_value() { + getter_method_plain_data_does_not_tolerate_none_value!("rate_pack_limits"); + } + + #[test] + #[should_panic( + expected = "Syntax error in rate_pack_limits value 'Booga!': should be -|-|-|- where L is low, H is high, R is routing, E is exit, BR is byte rate, and SR is service rate. All numbers should be in wei." + )] + fn rate_pack_limits_panics_at_syntax_error_in_value() { + persistent_config_plain_data_assertions_for_simple_get_method!( + "rate_pack_limits", + "Booga!", + // Irrelevant but necessary + RatePackLimits::new( + RatePack { + routing_byte_rate: 0, + routing_service_rate: 0, + exit_byte_rate: 0, + exit_service_rate: 0, + }, + RatePack { + routing_byte_rate: 0, + routing_service_rate: 0, + exit_byte_rate: 0, + exit_service_rate: 0, + } + ) + ); + } + + #[test] + #[should_panic( + expected = "Rate pack limits should have low limits less than high limits, but routing_byte_rate limits are 1-1" + )] + fn rate_pack_limits_panics_when_routing_byte_rate_limits_are_reversed() { + persistent_config_plain_data_assertions_for_simple_get_method!( + "rate_pack_limits", + "1-1|0-1|0-1|0-1", + // Irrelevant but necessary + RatePackLimits::new( + RatePack { + routing_byte_rate: 0, + routing_service_rate: 0, + exit_byte_rate: 0, + exit_service_rate: 0, + }, + RatePack { + routing_byte_rate: 0, + routing_service_rate: 0, + exit_byte_rate: 0, + exit_service_rate: 0, + } + ) + ); + } + + #[test] + #[should_panic( + expected = "Rate pack limits should have low limits less than high limits, but routing_service_rate limits are 1-1" + )] + fn rate_pack_limits_panics_when_routing_service_rate_limits_are_reversed() { + persistent_config_plain_data_assertions_for_simple_get_method!( + "rate_pack_limits", + "0-1|1-1|0-1|0-1", + // Irrelevant but necessary + RatePackLimits::new( + RatePack { + routing_byte_rate: 0, + routing_service_rate: 0, + exit_byte_rate: 0, + exit_service_rate: 0, + }, + RatePack { + routing_byte_rate: 0, + routing_service_rate: 0, + exit_byte_rate: 0, + exit_service_rate: 0, + } + ) + ); + } + + #[test] + #[should_panic( + expected = "Rate pack limits should have low limits less than high limits, but exit_byte_rate limits are 1-1" + )] + fn rate_pack_limits_panics_when_exit_byte_rate_limits_are_reversed() { + persistent_config_plain_data_assertions_for_simple_get_method!( + "rate_pack_limits", + "0-1|0-1|1-1|0-1", + // Irrelevant but necessary + RatePackLimits::new( + RatePack { + routing_byte_rate: 0, + routing_service_rate: 0, + exit_byte_rate: 0, + exit_service_rate: 0, + }, + RatePack { + routing_byte_rate: 0, + routing_service_rate: 0, + exit_byte_rate: 0, + exit_service_rate: 0, + } + ) + ); + } + + #[test] + #[should_panic( + expected = "Rate pack limits should have low limits less than high limits, but exit_service_rate limits are 1-1" + )] + fn rate_pack_limits_panics_when_exit_service_rate_limits_are_reversed() { + persistent_config_plain_data_assertions_for_simple_get_method!( + "rate_pack_limits", + "0-1|0-1|0-1|1-1", + // Irrelevant but necessary + RatePackLimits::new( + RatePack { + routing_byte_rate: 0, + routing_service_rate: 0, + exit_byte_rate: 0, + exit_service_rate: 0, + }, + RatePack { + routing_byte_rate: 0, + routing_service_rate: 0, + exit_byte_rate: 0, + exit_service_rate: 0, + } + ) + ); + } + #[test] fn scan_intervals_get_method_works() { persistent_config_plain_data_assertions_for_simple_get_method!( diff --git a/node/src/dispatcher.rs b/node/src/dispatcher.rs index ac698e043..4b01cd593 100644 --- a/node/src/dispatcher.rs +++ b/node/src/dispatcher.rs @@ -271,13 +271,13 @@ mod tests { let recording_arc = proxy_server.get_recording(); let awaiter = proxy_server.get_awaiter(); let client_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); - let reception_port = Some(8080); + let reception_port_opt = Some(8080); let data: Vec = vec![9, 10, 11]; let ibcd_in = InboundClientData { timestamp: SystemTime::now(), client_addr, - reception_port, - sequence_number: Some(0), + reception_port_opt, + sequence_number_opt: Some(0), last_data: false, is_clandestine: false, data: data.clone(), @@ -310,15 +310,15 @@ mod tests { let subject_addr = subject.start(); let (hopper, hopper_awaiter, hopper_recording_arc) = make_recorder(); let client_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); - let reception_port = Some(8080); + let reception_port_opt = Some(8080); let data: Vec = vec![9, 10, 11]; let ibcd_in = InboundClientData { timestamp: SystemTime::now(), client_addr, - reception_port, + reception_port_opt, last_data: false, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: data.clone(), }; let mut peer_actors = peer_actors_builder().hopper(hopper).build(); @@ -350,15 +350,15 @@ mod tests { let subject_addr = subject.start(); let subject_ibcd = subject_addr.recipient::(); let client_addr = SocketAddr::from_str("1.2.3.4:8765").unwrap(); - let reception_port = Some(1234); + let reception_port_opt = Some(1234); let data: Vec = vec![9, 10, 11]; let ibcd_in = InboundClientData { timestamp: SystemTime::now(), client_addr, - reception_port, + reception_port_opt, last_data: false, is_clandestine: false, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: data.clone(), }; @@ -376,15 +376,15 @@ mod tests { let subject_addr = subject.start(); let subject_ibcd = subject_addr.recipient::(); let client_addr = SocketAddr::from_str("1.2.3.4:8765").unwrap(); - let reception_port = Some(1234); + let reception_port_opt = Some(1234); let data: Vec = vec![9, 10, 11]; let ibcd_in = InboundClientData { timestamp: SystemTime::now(), client_addr, - reception_port, + reception_port_opt, last_data: false, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: data.clone(), }; @@ -406,7 +406,7 @@ mod tests { let obcd = TransmitDataMsg { endpoint: Endpoint::Socket(socket_addr), last_data: false, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: data.clone(), }; @@ -430,7 +430,7 @@ mod tests { let obcd = TransmitDataMsg { endpoint: Endpoint::Socket(socket_addr), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: data.clone(), }; let mut peer_actors = peer_actors_builder().build(); diff --git a/node/src/hopper/consuming_service.rs b/node/src/hopper/consuming_service.rs index 7d5660a8b..0eb34cb52 100644 --- a/node/src/hopper/consuming_service.rs +++ b/node/src/hopper/consuming_service.rs @@ -100,10 +100,10 @@ impl ConsumingService { let ibcd = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 0), - reception_port: None, + reception_port_opt: None, last_data: false, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: encrypted_package.into(), }; debug!( @@ -119,7 +119,7 @@ impl ConsumingService { endpoint: next_stop, last_data: false, // Hopper-to-Hopper clandestine streams are never remotely killed data: encrypted_package.into(), - sequence_number: None, + sequence_number_opt: None, }; debug!( @@ -143,6 +143,7 @@ mod tests { use crate::sub_lib::node_addr::NodeAddr; use crate::sub_lib::route::Route; use crate::sub_lib::route::RouteSegment; + use crate::sub_lib::stream_key::StreamKey; use crate::test_utils::recorder::make_recorder; use crate::test_utils::recorder::peer_actors_builder; use crate::test_utils::{make_meaningless_message_type, make_paying_wallet}; @@ -168,7 +169,7 @@ mod tests { CRYPTDE_PAIR.main.as_ref(), &target_key, &target_node_addr, - make_meaningless_message_type(), + make_meaningless_message_type(StreamKey::make_meaningless_stream_key()), ) .unwrap(); let system = System::new(""); @@ -192,7 +193,7 @@ mod tests { &TransmitDataMsg { endpoint: Endpoint::Socket(SocketAddr::from_str("1.2.1.2:1212").unwrap()), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: encodex(CRYPTDE_PAIR.main.as_ref(), &target_key, &lcp) .unwrap() .into(), @@ -242,7 +243,7 @@ mod tests { Some(TEST_DEFAULT_CHAIN.rec().contract), ) .unwrap(); - let payload = make_meaningless_message_type(); + let payload = make_meaningless_message_type(StreamKey::make_meaningless_stream_key()); let incipient_cores_package = IncipientCoresPackage::new(cryptde, route.clone(), payload, &destination_key).unwrap(); let system = System::new("converts_incipient_message_to_live_and_sends_to_dispatcher"); @@ -266,7 +267,7 @@ mod tests { TransmitDataMsg { endpoint: Endpoint::Key(destination_key.clone()), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: expected_lcp_enc.into(), }, *record, @@ -289,7 +290,7 @@ mod tests { Some(TEST_DEFAULT_CHAIN.rec().contract), ) .unwrap(); - let payload = make_meaningless_message_type(); + let payload = make_meaningless_message_type(StreamKey::make_meaningless_stream_key()); let incipient_cores_package = IncipientCoresPackage::new(cryptde, route.clone(), payload, &destination_key).unwrap(); let system = System::new("consume_sends_zero_hop_incipient_directly_to_hopper"); @@ -317,10 +318,10 @@ mod tests { InboundClientData { timestamp: record.timestamp, client_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 0), - reception_port: None, + reception_port_opt: None, last_data: false, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: expected_lcp_enc.into(), }, ); @@ -340,7 +341,7 @@ mod tests { IncipientCoresPackage::new( CRYPTDE_PAIR.main.as_ref(), Route { hops: vec![] }, - make_meaningless_message_type(), + make_meaningless_message_type(StreamKey::make_meaningless_stream_key()), &PublicKey::new(&[1, 2]), ) .unwrap(), diff --git a/node/src/hopper/live_cores_package.rs b/node/src/hopper/live_cores_package.rs index 2b05b27e8..cac27bebc 100644 --- a/node/src/hopper/live_cores_package.rs +++ b/node/src/hopper/live_cores_package.rs @@ -100,6 +100,7 @@ mod tests { use crate::sub_lib::node_addr::NodeAddr; use crate::sub_lib::route::RouteSegment; use crate::sub_lib::route::{Route, RouteError}; + use crate::sub_lib::stream_key::StreamKey; use crate::test_utils::{ make_meaningless_message_type, make_meaningless_route, make_paying_wallet, }; @@ -141,7 +142,9 @@ mod tests { let relay_key = PublicKey::new(&[1, 2]); let relay_cryptde = CryptDENull::from(&relay_key, TEST_DEFAULT_CHAIN); let cryptde = CRYPTDE_PAIR.main.as_ref(); - let serialized_payload = serde_cbor::ser::to_vec(&make_meaningless_message_type()).unwrap(); + let stream_key = StreamKey::make_meaningless_stream_key(); + let serialized_payload = + serde_cbor::ser::to_vec(&make_meaningless_message_type(stream_key)).unwrap(); let encrypted_payload = cryptde .encode(&destination_key, &PlainData::new(&serialized_payload)) .unwrap(); @@ -204,7 +207,7 @@ mod tests { let key34 = PublicKey::new(&[3, 4]); let node_addr34 = NodeAddr::new(&IpAddr::from_str("3.4.3.4").unwrap(), &[1234]); let mut route = Route::single_hop(&key34, cryptde).unwrap(); - let payload = make_meaningless_message_type(); + let payload = make_meaningless_message_type(StreamKey::make_meaningless_stream_key()); let incipient = NoLookupIncipientCoresPackage::new(cryptde, &key34, &node_addr34, payload.clone()) @@ -228,7 +231,7 @@ mod tests { cryptde, &blank_key, &node_addr34, - make_meaningless_message_type(), + make_meaningless_message_type(StreamKey::make_meaningless_stream_key()), ); assert_eq!( @@ -254,7 +257,7 @@ mod tests { Some(contract_address), ) .unwrap(); - let payload = make_meaningless_message_type(); + let payload = make_meaningless_message_type(StreamKey::make_meaningless_stream_key()); let incipient = IncipientCoresPackage::new(cryptde, route.clone(), payload.clone(), &key56).unwrap(); @@ -280,7 +283,7 @@ mod tests { let incipient = IncipientCoresPackage::new( cryptde, Route { hops: vec![] }, - make_meaningless_message_type(), + make_meaningless_message_type(StreamKey::make_meaningless_stream_key()), &PublicKey::new(&[3, 4]), ) .unwrap(); @@ -297,7 +300,8 @@ mod tests { #[test] fn expired_cores_package_can_be_constructed_from_live_cores_package() { let immediate_neighbor_ip = SocketAddr::from_str("1.2.3.4:1234").unwrap(); - let payload = make_meaningless_message_type(); + let stream_key = StreamKey::make_meaningless_stream_key(); + let payload = make_meaningless_message_type(stream_key); let first_stop_key = PublicKey::new(&[3, 4]); let first_stop_cryptde = CryptDENull::from(&first_stop_key, TEST_DEFAULT_CHAIN); let relay_key = PublicKey::new(&[1, 2]); @@ -316,7 +320,6 @@ mod tests { ), cryptde, Some(paying_wallet.clone()), - 1234, Some(contract_address), ) .unwrap(); @@ -375,11 +378,6 @@ mod tests { Component::ProxyServer, ) ); - assert_eq!( - route.hops[0], - crate::test_utils::encrypt_return_route_id(1234, cryptde), - ); - route.hops.remove(0); assert_eq!( &route.hops[0].as_slice()[..8], &[52, 52, 52, 52, 52, 52, 52, 52] diff --git a/node/src/hopper/mod.rs b/node/src/hopper/mod.rs index 588016440..b2f3d6dc0 100644 --- a/node/src/hopper/mod.rs +++ b/node/src/hopper/mod.rs @@ -149,6 +149,7 @@ mod tests { use crate::sub_lib::hopper::IncipientCoresPackage; use crate::sub_lib::route::Route; use crate::sub_lib::route::RouteSegment; + use crate::sub_lib::stream_key::StreamKey; use crate::test_utils::unshared_test_utils::prove_that_crash_request_handler_is_hooked_up; use crate::test_utils::{ make_meaningless_message_type, make_paying_wallet, route_to_proxy_client, @@ -175,8 +176,10 @@ mod tests { fn panics_if_routing_service_is_unbound() { let main_cryptde = CRYPTDE_PAIR.main.as_ref(); let client_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); - let route = route_to_proxy_client(&main_cryptde.public_key(), main_cryptde); - let serialized_payload = serde_cbor::ser::to_vec(&make_meaningless_message_type()).unwrap(); + let route = route_to_proxy_client(&main_cryptde.public_key(), main_cryptde, false); + let stream_key = StreamKey::make_meaningless_stream_key(); + let serialized_payload = + serde_cbor::ser::to_vec(&make_meaningless_message_type(stream_key)).unwrap(); let data = main_cryptde .encode( &main_cryptde.public_key(), @@ -193,10 +196,10 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr, - reception_port: None, + reception_port_opt: None, last_data: false, is_clandestine: false, - sequence_number: None, + sequence_number_opt: None, data: encrypted_package, }; let system = System::new("panics_if_routing_service_is_unbound"); @@ -234,7 +237,7 @@ mod tests { let incipient_package = IncipientCoresPackage::new( main_cryptde, route, - make_meaningless_message_type(), + make_meaningless_message_type(StreamKey::make_meaningless_stream_key()), &main_cryptde.public_key(), ) .unwrap(); diff --git a/node/src/hopper/routing_service.rs b/node/src/hopper/routing_service.rs index 1ad3ba578..e2f637820 100644 --- a/node/src/hopper/routing_service.rs +++ b/node/src/hopper/routing_service.rs @@ -186,10 +186,10 @@ impl RoutingService { let inbound_client_data = InboundClientData { timestamp: ibcd_but_data.timestamp, client_addr: ibcd_but_data.client_addr, - reception_port: ibcd_but_data.reception_port, + reception_port_opt: ibcd_but_data.reception_port_opt, last_data: ibcd_but_data.last_data, is_clandestine: ibcd_but_data.is_clandestine, - sequence_number: ibcd_but_data.sequence_number, + sequence_number_opt: ibcd_but_data.sequence_number_opt, data: payload.into(), }; self.routing_service_subs @@ -495,7 +495,7 @@ impl RoutingService { endpoint: Endpoint::Key(next_hop.public_key), last_data, data: next_live_package_enc.into(), - sequence_number: None, + sequence_number_opt: None, }) } } @@ -542,16 +542,18 @@ mod tests { #[test] fn dns_resolution_failures_are_reported_to_the_proxy_server() { - let cryptde_pair = CRYPTDE_PAIR.clone(); - let route = - route_to_proxy_server(&cryptde_pair.main.public_key(), cryptde_pair.main.as_ref()); + let route = route_to_proxy_server( + &CRYPTDE_PAIR.main.public_key(), + CRYPTDE_PAIR.main.as_ref(), + false, + ); let stream_key = StreamKey::make_meaningless_stream_key(); let dns_resolve_failure = DnsResolveFailure_0v1::new(stream_key); let lcp = LiveCoresPackage::new( route, encodex( - cryptde_pair.alias.as_ref(), - &cryptde_pair.alias.public_key(), + CRYPTDE_PAIR.alias.as_ref(), + &CRYPTDE_PAIR.alias.public_key(), &MessageType::DnsResolveFailed(VersionedData::new( &crate::sub_lib::migrations::dns_resolve_failure::MIGRATIONS, &dns_resolve_failure.clone(), @@ -560,16 +562,16 @@ mod tests { .unwrap(), ); let data_enc = encodex( - cryptde_pair.main.as_ref(), - &cryptde_pair.main.public_key(), + CRYPTDE_PAIR.main.as_ref(), + &CRYPTDE_PAIR.main.public_key(), &lcp, ) .unwrap(); let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, - sequence_number: None, + reception_port_opt: None, + sequence_number_opt: None, last_data: false, is_clandestine: false, data: data_enc.into(), @@ -579,7 +581,7 @@ mod tests { let system = System::new("dns_resolution_failures_are_reported_to_the_proxy_server"); let peer_actors = peer_actors_builder().proxy_server(proxy_server).build(); let subject = RoutingService::new( - cryptde_pair, + CRYPTDE_PAIR.clone(), RoutingServiceSubs { proxy_client_subs_opt: peer_actors.proxy_client_opt, proxy_server_subs: peer_actors.proxy_server, @@ -607,36 +609,38 @@ mod tests { fn logs_and_ignores_message_that_cannot_be_deserialized() { init_test_logging(); let test_name = "logs_and_ignores_message_that_cannot_be_deserialized"; - let cryptde_pair = CRYPTDE_PAIR.clone(); - let route = - route_from_proxy_client(&cryptde_pair.main.public_key(), cryptde_pair.main.as_ref()); + let route = route_from_proxy_client( + &CRYPTDE_PAIR.main.public_key(), + CRYPTDE_PAIR.main.as_ref(), + false, + ); let lcp = LiveCoresPackage::new( route, encodex( - cryptde_pair.main.as_ref(), - &cryptde_pair.main.public_key(), + CRYPTDE_PAIR.main.as_ref(), + &CRYPTDE_PAIR.main.public_key(), &[42u8], ) .unwrap(), ); let data_enc = encodex( - cryptde_pair.main.as_ref(), - &cryptde_pair.main.public_key(), + CRYPTDE_PAIR.main.as_ref(), + &CRYPTDE_PAIR.main.public_key(), &lcp, ) .unwrap(); let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, - sequence_number: None, + reception_port_opt: None, + sequence_number_opt: None, last_data: false, is_clandestine: false, data: data_enc.into(), }; let peer_actors = peer_actors_builder().build(); let mut subject = RoutingService::new( - cryptde_pair, + CRYPTDE_PAIR.clone(), RoutingServiceSubs { proxy_client_subs_opt: peer_actors.proxy_client_opt, proxy_server_subs: peer_actors.proxy_server, @@ -664,7 +668,7 @@ mod tests { let test_name = "logs_and_ignores_message_that_cannot_be_decrypted"; let main_cryptde = CryptDEReal::new(TEST_DEFAULT_CHAIN); let rogue_cryptde = CryptDEReal::new(TEST_DEFAULT_CHAIN); - let route = route_from_proxy_client(main_cryptde.public_key(), &main_cryptde); + let route = route_from_proxy_client(main_cryptde.public_key(), &main_cryptde, false); let lcp = LiveCoresPackage::new( route, encodex(&rogue_cryptde, rogue_cryptde.public_key(), &[42u8]).unwrap(), @@ -673,8 +677,8 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, - sequence_number: None, + reception_port_opt: None, + sequence_number_opt: None, last_data: false, is_clandestine: false, data: data_enc.into(), @@ -711,7 +715,7 @@ mod tests { fn logs_and_ignores_message_that_had_invalid_destination() { init_test_logging(); let main_cryptde = CRYPTDE_PAIR.main.as_ref(); - let route = route_from_proxy_client(&main_cryptde.public_key(), main_cryptde); + let route = route_from_proxy_client(&main_cryptde.public_key(), main_cryptde, false); let payload = GossipBuilder::empty(); let lcp = LiveCoresPackage::new( route, @@ -726,8 +730,8 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, - sequence_number: None, + reception_port_opt: None, + sequence_number_opt: None, last_data: false, is_clandestine: false, data: data_enc.into(), @@ -757,7 +761,7 @@ mod tests { BAN_CACHE.clear(); let main_cryptde = CRYPTDE_PAIR.main.as_ref(); let (component, _, component_recording_arc) = make_recorder(); - let route = route_to_proxy_client(&main_cryptde.public_key(), main_cryptde); + let route = route_to_proxy_client(&main_cryptde.public_key(), main_cryptde, false); let payload = make_request_payload(0, main_cryptde); let lcp = LiveCoresPackage::new( route, @@ -776,8 +780,8 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, - sequence_number: None, + reception_port_opt: None, + sequence_number_opt: None, last_data: true, is_clandestine: false, data: data_enc.into(), @@ -828,7 +832,7 @@ mod tests { BAN_CACHE.clear(); let test_name = "complains_about_live_message_for_nonexistent_proxy_client"; let main_cryptde = CRYPTDE_PAIR.main.as_ref(); - let route = route_to_proxy_client(&main_cryptde.public_key(), main_cryptde); + let route = route_to_proxy_client(&main_cryptde.public_key(), main_cryptde, false); let payload = make_request_payload(0, main_cryptde); let lcp = LiveCoresPackage::new( route, @@ -846,8 +850,8 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, - sequence_number: None, + reception_port_opt: None, + sequence_number_opt: None, last_data: true, is_clandestine: false, data: data_enc.into(), @@ -887,7 +891,7 @@ mod tests { let main_cryptde = CRYPTDE_PAIR.main.as_ref(); let alias_cryptde = CRYPTDE_PAIR.alias.as_ref(); let (proxy_server, _, proxy_server_recording_arc) = make_recorder(); - let route = route_to_proxy_server(&main_cryptde.public_key(), main_cryptde); + let route = route_to_proxy_server(&main_cryptde.public_key(), main_cryptde, false); let payload = make_response_payload(0); let lcp = LiveCoresPackage::new( route, @@ -903,10 +907,10 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.3.2.4:5678").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: false, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: lcp_enc.into(), }; @@ -980,10 +984,10 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.3.2.4:5678").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: false, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: data_enc.into(), }; @@ -1053,10 +1057,10 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.3.2.4:5678").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: false, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: data_enc.into(), }; @@ -1129,10 +1133,10 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: true, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: data_enc.into(), }; @@ -1172,7 +1176,7 @@ mod tests { TransmitDataMsg { endpoint: Endpoint::Key(next_key.clone()), last_data: true, - sequence_number: None, + sequence_number_opt: None, data: expected_lcp_enc.into(), } ); @@ -1221,10 +1225,10 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: true, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: data_enc.into(), }; @@ -1264,10 +1268,10 @@ mod tests { InboundClientData { timestamp: record.timestamp, client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: true, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: expected_lcp_enc.into() } ); @@ -1284,7 +1288,7 @@ mod tests { let origin_key = PublicKey::new(&[1, 2]); let origin_cryptde = CryptDENull::from(&origin_key, TEST_DEFAULT_CHAIN); let destination_key = PublicKey::new(&[3, 4]); - let payload = make_meaningless_message_type(); + let payload = make_meaningless_message_type(StreamKey::make_meaningless_stream_key()); let route = Route::one_way( RouteSegment::new( vec![&origin_key, &main_cryptde.public_key(), &destination_key], @@ -1305,10 +1309,10 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: true, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: data_enc.into(), }; let system = System::new(test_name); @@ -1402,10 +1406,10 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: true, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: data_enc.into(), }; let system = System::new(test_name); @@ -1579,10 +1583,10 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: true, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: data_enc.into(), }; let system = System::new("test"); @@ -1652,10 +1656,10 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: true, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: data_enc.into(), }; let system = System::new("test"); @@ -1699,10 +1703,10 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: true, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: vec![], }; let system = System::new("consume_logs_error_when_given_bad_input_data"); @@ -1758,10 +1762,10 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: true, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: data_enc.into(), }; let system = System::new("consume_logs_error_when_given_bad_input_data"); @@ -1828,10 +1832,10 @@ mod tests { let ibcd = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: true, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: vec![], }; @@ -1893,7 +1897,7 @@ mod tests { &ClientRequestPayload_0v1 { stream_key: StreamKey::make_meaningless_stream_key(), sequenced_packet: SequencedPacket::new(vec![1, 2, 3, 4], 1234, false), - target_hostname: Some("hostname".to_string()), + target_hostname: "hostname".to_string(), target_port: 1234, protocol: ProxyProtocol::TLS, originator_public_key: PublicKey::new(b"1234"), diff --git a/node/src/neighborhood/dot_graph.rs b/node/src/neighborhood/dot_graph.rs index 78de9f339..e6ae6f056 100644 --- a/node/src/neighborhood/dot_graph.rs +++ b/node/src/neighborhood/dot_graph.rs @@ -18,7 +18,7 @@ pub struct NodeRenderableInner { pub struct NodeRenderable { pub inner: Option, pub public_key: PublicKey, - pub node_addr: Option, + pub node_addr_opt: Option, pub known_source: bool, pub known_target: bool, pub is_present: bool, @@ -60,7 +60,7 @@ impl NodeRenderable { } else { &public_key_str }; - let node_addr_string = match self.node_addr { + let node_addr_string = match self.node_addr_opt { None => String::new(), Some(ref na) => format!("\\n{}", na), }; @@ -112,7 +112,7 @@ mod tests { country_code: "ZZ".to_string(), }), public_key: public_key.clone(), - node_addr: None, + node_addr_opt: None, known_source: false, known_target: false, is_present: true, @@ -141,7 +141,7 @@ mod tests { country_code: "ZZ".to_string(), }), public_key: public_key.clone(), - node_addr: None, + node_addr_opt: None, known_source: false, known_target: false, is_present: true, diff --git a/node/src/neighborhood/gossip.rs b/node/src/neighborhood/gossip.rs index 15daeaf91..8381b1d80 100644 --- a/node/src/neighborhood/gossip.rs +++ b/node/src/neighborhood/gossip.rs @@ -14,16 +14,20 @@ use serde_cbor::Value; use serde_derive::{Deserialize, Serialize}; use std::collections::HashSet; use std::convert::{TryFrom, TryInto}; -use std::fmt::Debug; use std::fmt::Error; use std::fmt::Formatter; use std::fmt::Write as _; +use std::fmt::{Debug, Display}; use std::net::{IpAddr, SocketAddr}; #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct GossipNodeRecord { pub signed_data: PlainData, pub signature: CryptData, + // This field should probably not exist. If the Gossip source is our next-door neighbor, + // we can get its IP address from the network stack, which is more reliable than an + // unsigned data field. If the Gossip source is not our next-door neighbor, we should + // not know its IP address anyway. pub node_addr_opt: Option, } @@ -389,7 +393,7 @@ impl Gossip_0v1 { country_code, }), public_key: nri.public_key.clone(), - node_addr: addr.clone(), + node_addr_opt: addr.clone(), known_source: nri.public_key == source.public_key, known_target: nri.public_key == target.public_key, is_present: true, @@ -399,7 +403,7 @@ impl Gossip_0v1 { node_renderables.push(NodeRenderable { inner: None, public_key: k.clone(), - node_addr: None, + node_addr_opt: None, known_source: false, known_target: false, is_present: false, @@ -472,6 +476,28 @@ pub struct AccessibleGossipRecord { pub inner: NodeRecordInner_0v1, } +impl Display for AccessibleGossipRecord { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self.node_addr_opt { + Some(ref addr) => { + write!( + f, + "{} {} {} {} {} {} {} {}", + self.inner.behavior_string(), + self.inner.version_string(), + self.inner.country_code_string(), + self.inner.public_key_string(), + addr.ip_addr(), + self.inner.wallet_string(), + self.inner.rate_pack_string(), + self.inner.neighbors_string() + ) + } + None => write!(f, "{}", self.inner), + } + } +} + impl AccessibleGossipRecord { pub fn regenerate_signed_gossip(&mut self, cryptde: &dyn CryptDE) { let (signed_gossip, signature) = regenerate_signed_gossip(&self.inner, cryptde); @@ -605,6 +631,52 @@ mod tests { assert_eq!(gossip.node_records.remove(0).node_addr_opt, None) } + #[test] + fn accessible_gossip_record_to_string_without_ip() { + let mut node_record = make_node_record(1234, true); + node_record.inner.country_code_opt = Some("AD".to_string()); + let db = db_from_node(&node_record); + let subject = AccessibleGossipRecord::from((&db, node_record.public_key(), false)); + + let result = subject.to_string(); + + assert_eq!( + result, + "AR v0 AD AQIDBA 0x546900db8d6e0937497133d1ae6fdf5f4b75bcd0 1235|1434|1237|1634 []" + ); + } + + #[test] + fn accessible_gossip_record_to_string_with_ip() { + let mut node_record = make_node_record(1234, true); + node_record.inner.country_code_opt = Some("AD".to_string()); + let db = db_from_node(&node_record); + let subject = AccessibleGossipRecord::from((&db, node_record.public_key(), true)); + + let result = subject.to_string(); + + assert_eq!( + result, + "AR v0 AD AQIDBA 1.2.3.4 0x546900db8d6e0937497133d1ae6fdf5f4b75bcd0 1235|1434|1237|1634 []" + ); + } + + #[test] + fn accessible_gossip_record_to_string_with_ip_and_really_big_version() { + let mut node_record = make_node_record(1234, true); + node_record.inner.country_code_opt = Some("AD".to_string()); + let db = db_from_node(&node_record); + let mut subject = AccessibleGossipRecord::from((&db, node_record.public_key(), true)); + subject.inner.version = 4_000_000_000; + + let result = subject.to_string(); + + assert_eq!( + result, + "AR v4000000000 AD AQIDBA 1.2.3.4 0x546900db8d6e0937497133d1ae6fdf5f4b75bcd0 1235|1434|1237|1634 []" + ); + } + #[test] fn gossip_node_record_keeps_all_half_neighbors() { let mut this_node = make_node_record(1234, true); diff --git a/node/src/neighborhood/gossip_acceptor.rs b/node/src/neighborhood/gossip_acceptor.rs index f99eba27d..4699a7e88 100644 --- a/node/src/neighborhood/gossip_acceptor.rs +++ b/node/src/neighborhood/gossip_acceptor.rs @@ -1,17 +1,22 @@ // Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. +use crate::db_config::persistent_configuration::PersistentConfiguration; use crate::neighborhood::gossip::{ AccessibleGossipRecord, GossipBuilder, GossipNodeRecord, Gossip_0v1, }; +use crate::neighborhood::malefactor::Malefactor; use crate::neighborhood::neighborhood_database::{NeighborhoodDatabase, NeighborhoodDatabaseError}; use crate::neighborhood::node_record::NodeRecord; use crate::neighborhood::UserExitPreferences; use crate::sub_lib::cryptde::{CryptDE, PublicKey}; use crate::sub_lib::neighborhood::{ ConnectionProgressEvent, ConnectionProgressMessage, GossipFailure_0v1, NeighborhoodMetadata, + RatePackLimits, }; use crate::sub_lib::node_addr::NodeAddr; +use itertools::Itertools; use masq_lib::logger::Logger; +use std::any::Any; use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, SocketAddr}; @@ -26,6 +31,7 @@ pub const MAX_DEGREE: usize = 5; const PASS_GOSSIP_EXPIRED_TIME: Duration = Duration::from_secs(60); #[derive(Clone, PartialEq, Eq, Debug)] +#[allow(clippy::large_enum_variant)] pub enum GossipAcceptanceResult { // The incoming Gossip produced database changes. Generate standard Gossip and broadcast. Accepted, @@ -33,17 +39,16 @@ pub enum GossipAcceptanceResult { Reply(Gossip_0v1, PublicKey, NodeAddr), // The incoming Gossip was proper, and we tried to accept it, but couldn't. Failed(GossipFailure_0v1, PublicKey, NodeAddr), - // The incoming Gossip contained nothing we didn't know. Don't send out any Gossip because of it. - Ignored, // Gossip was ignored because it was evil: ban the sender of the Gossip as a malefactor. - Ban(String), + Ban(Malefactor), } #[derive(Clone, PartialEq, Eq, Debug)] +#[allow(clippy::large_enum_variant)] enum Qualification { Matched, - Unmatched, - Malformed(String), + Unmatched(String), + Malformed(Malefactor), } trait NamedType { @@ -64,10 +69,13 @@ trait GossipHandler: NamedType + Send /* Send because lazily-written tests requi agrs: Vec, gossip_source: SocketAddr, neighborhood_metadata: NeighborhoodMetadata, - ) -> GossipAcceptanceResult; + ) -> Vec; + + fn as_any(&self) -> &dyn Any; } struct DebutHandler { + rate_pack_limits: RatePackLimits, logger: Logger, } @@ -88,34 +96,23 @@ impl GossipHandler for DebutHandler { gossip_source: SocketAddr, ) -> Qualification { if agrs.len() != 1 { - return Qualification::Unmatched; + return Qualification::Unmatched(format!( + "Debut has 1 Node record, not {}", + agrs.len() + )); } if database.node_by_key(&agrs[0].inner.public_key).is_some() { - return Qualification::Unmatched; + return Qualification::Unmatched("Node record already in database".to_string()); } - //TODO: Create optimization card to drive in the following logic: - // Imagine a brand-new network, consisting only of Node A. - // When Node B debuts, Node A cannot respond with an Introduction, - // since there's nobody to introduce. Therefore, Node A must - // respond with a single-Node Gossip that will currently be - // interpreted as a Debut by Node B, resulting in another - // single-Node Gossip from B to A. This last Gossip isn't a - // problem, because Node A will ignore it since B is already - // in its database. However, there is a possible optimization: - // drive in the code below, and Node B will no longer interpret - // Node A's acceptance Gossip as another Debut, because it will - // see that Node A already has Node B as a neighbor. This means - // Node A's original response will be interpreted as Standard - // Gossip. - // if agrs[0].inner.neighbors.contains(database.root_key()) { - // return Qualification::Unmatched; - // } match &agrs[0].node_addr_opt { None => { if agrs[0].inner.accepts_connections { - Qualification::Malformed(format!( - "Debut from {} for {} contained no NodeAddr", - gossip_source, agrs[0].inner.public_key + Qualification::Malformed(Malefactor::new( + Some(agrs[0].inner.public_key.clone()), + Some(gossip_source.ip()), + Some(agrs[0].inner.earning_wallet.clone()), + None, + "Debut contained no NodeAddr".to_string(), )) } else { Qualification::Matched @@ -124,19 +121,25 @@ impl GossipHandler for DebutHandler { Some(node_addr) => { if agrs[0].inner.accepts_connections { if node_addr.ports().is_empty() { - Qualification::Malformed(format!( - "Debut from {} for {} contained NodeAddr with no ports", - gossip_source, agrs[0].inner.public_key + Qualification::Malformed(Malefactor::new( + Some(agrs[0].inner.public_key.clone()), + Some(gossip_source.ip()), + Some(agrs[0].inner.earning_wallet.clone()), + None, + "Debut contained NodeAddr with no ports".to_string(), )) } else if node_addr.ip_addr() == gossip_source.ip() { Qualification::Matched } else { - Qualification::Unmatched + Qualification::Unmatched("Node record is not Gossip source".to_string()) } } else { - Qualification::Malformed(format!( - "Debut from {} for {} does not accept connections, yet contained NodeAddr", - gossip_source, agrs[0].inner.public_key + Qualification::Malformed(Malefactor::new( + Some(agrs[0].inner.public_key.clone()), + Some(gossip_source.ip()), + Some(agrs[0].inner.earning_wallet.clone()), + None, + "Debut does not accept connections, yet contained NodeAddr".to_string(), )) } } @@ -150,7 +153,7 @@ impl GossipHandler for DebutHandler { mut agrs: Vec, gossip_source: SocketAddr, neighborhood_metadata: NeighborhoodMetadata, - ) -> GossipAcceptanceResult { + ) -> Vec { let source_agr = { let mut agr = agrs.remove(0); // empty Gossip shouldn't get here if agr.node_addr_opt.is_none() { @@ -158,6 +161,28 @@ impl GossipHandler for DebutHandler { } agr }; + if let Err(message) = GossipAcceptorReal::validate_incoming_agr( + &source_agr, + format!( + "Debut from {} at {}", + source_agr.inner.public_key, gossip_source + ), + &self.rate_pack_limits, + &self.logger, + ) { + return vec![GossipAcceptanceResult::Ban(Malefactor::new( + Some(source_agr.inner.public_key.clone()), + Some( + source_agr + .node_addr_opt + .expect("IP address disappeared") + .ip_addr(), + ), + Some(source_agr.inner.earning_wallet), + None, + message, + ))]; + } let source_key = source_agr.inner.public_key.clone(); let source_node_addr = source_agr .node_addr_opt @@ -178,11 +203,11 @@ impl GossipHandler for DebutHandler { preferred_key, preferred_ip, ); - return GossipAcceptanceResult::Reply( + return vec![GossipAcceptanceResult::Reply( Self::make_pass_gossip(database, preferred_key), source_key, source_node_addr, - ); + )]; } if let Ok(result) = self.try_accept_debut( cryptde, @@ -191,7 +216,7 @@ impl GossipHandler for DebutHandler { gossip_source, neighborhood_metadata.user_exit_preferences_opt, ) { - return result; + return vec![result]; } debug!(self.logger, "Seeking neighbor for Pass"); let lcn_key = match Self::find_least_connected_half_neighbor_excluding( @@ -203,14 +228,14 @@ impl GossipHandler for DebutHandler { "Neighbor count at maximum, but no non-common neighbors. DebutHandler is reluctantly ignoring debut from {} at {}", source_key, source_node_addr ); - return GossipAcceptanceResult::Failed( + return vec![GossipAcceptanceResult::Failed( GossipFailure_0v1::NoSuitableNeighbors, (&source_agr.inner.public_key).clone(), (&source_agr .node_addr_opt .expect("Debuter's NodeAddr disappeared")) .clone(), - ); + )]; } Some(key) => key, }; @@ -230,17 +255,29 @@ impl GossipHandler for DebutHandler { lcn_key, lcn_ip_str ); - GossipAcceptanceResult::Reply( + vec![GossipAcceptanceResult::Reply( Self::make_pass_gossip(database, lcn_key), source_key, source_node_addr, - ) + )] + } + + fn as_any(&self) -> &dyn Any { + self } } +enum IntroductionAttempt { + Success(Gossip_0v1, PublicKey, NodeAddr), + Failure, +} + impl DebutHandler { - fn new(logger: Logger) -> DebutHandler { - DebutHandler { logger } + fn new(rate_pack_limits: &RatePackLimits, logger: Logger) -> DebutHandler { + DebutHandler { + rate_pack_limits: rate_pack_limits.clone(), + logger, + } } fn find_more_appropriate_neighbor<'b>( @@ -253,10 +290,8 @@ impl DebutHandler { let qualified_neighbors: Vec<&PublicKey> = neighbor_vec .into_iter() .filter(|k| { - database - .node_by_key(*k) - .expect("Node disappeared") - .accepts_connections() + let node = database.node_by_key(*k).expect("Node disappeared"); + node.accepts_connections() && node.routes_data() }) .skip_while(|k| database.gossip_target_degree(*k) <= 2) .collect(); @@ -321,50 +356,93 @@ impl DebutHandler { root_mut.increment_version(); root_mut.regenerate_signed_gossip(cryptde); trace!(self.logger, "Current database: {}", database.to_dot_graph()); - if Self::should_not_make_another_introduction(debuting_agr) { + if Self::should_not_make_another_introduction(database, debuting_agr) { let ip_addr_str = match &debuting_agr.node_addr_opt { Some(node_addr) => node_addr.ip_addr().to_string(), None => "?.?.?.?".to_string(), }; - debug!(self.logger, "Node {} at {} is responding to first introduction: sending standard Gossip instead of further introduction", - debuting_agr.inner.public_key, - ip_addr_str); - Ok(GossipAcceptanceResult::Accepted) - } else { - match self.make_introduction(database, debuting_agr, gossip_source) { - Some((introduction, target_key, target_node_addr)) => { - Ok(GossipAcceptanceResult::Reply( - introduction, - target_key, - target_node_addr, - )) + if Self::counts_us_among_neighbors(debuting_agr, database) { + debug!( + self.logger, + "Can't send introduction to Node {} at {}, but it already knows us; sending Standard Gossip", + &debut_node_key, + gossip_source, + ); + Ok(GossipAcceptanceResult::Accepted) + } else { + match GossipAcceptorReal::make_debut_triple(database, debuting_agr) { + Ok((redebut_gossip, public_key, node_addr)) => { + debug!( + self.logger, + "Can't send introduction to Node {} at {}; sending redebut instead", + debuting_agr.inner.public_key, + ip_addr_str + ); + Ok(GossipAcceptanceResult::Reply( + redebut_gossip, + public_key, + node_addr, + )) + } + Err(e) => { + debug!( + self.logger, + "Can't send either introduction or redebut to Node {} at {} ({}); sending standard Gossip instead", + debuting_agr.inner.public_key, + ip_addr_str, + e + ); + Ok(GossipAcceptanceResult::Accepted) + } } - None => { - debug!( - self.logger, - "DebutHandler has no one to introduce, but is debuting back to {} at {}", - &debut_node_key, - gossip_source, - ); + } + } else { + match self.try_to_make_introduction(database, debuting_agr, gossip_source) { + IntroductionAttempt::Success( + introduction, + target_key, + target_node_addr, + ) => Ok(GossipAcceptanceResult::Reply( + introduction, + target_key, + target_node_addr, + )), + IntroductionAttempt::Failure => { trace!( self.logger, "DebutHandler database state: {}", &database.to_dot_graph(), ); - let debut_gossip = Self::create_debut_gossip_response( - cryptde, - database, - debut_node_key, - ); - Ok(GossipAcceptanceResult::Reply( - debut_gossip, - debuting_agr.inner.public_key.clone(), - debuting_agr - .node_addr_opt - .as_ref() - .expect("Debut gossip always has an IP") - .clone(), - )) + if Self::counts_us_among_neighbors(debuting_agr, database) { + debug!( + self.logger, + "DebutHandler has no one to introduce and debutant already has us as a neighbor; accepting {} at {} and sending Standard Gossip", + &debut_node_key, + gossip_source, + ); + Ok(GossipAcceptanceResult::Accepted) + } else { + debug!( + self.logger, + "DebutHandler has no one to introduce, but debutant does not have us as a neighbor; debuting back to {} at {}", + &debut_node_key, + gossip_source, + ); + let debut_gossip = Self::create_second_node_gossip_response( + cryptde, + database, + debut_node_key, + ); + Ok(GossipAcceptanceResult::Reply( + debut_gossip, + debuting_agr.inner.public_key.clone(), + debuting_agr + .node_addr_opt + .as_ref() + .expect("Debut gossip always has an IP") + .clone(), + )) + } } } } @@ -373,12 +451,23 @@ impl DebutHandler { } } - fn create_debut_gossip_response( + fn counts_us_among_neighbors( + debuting_agr: &AccessibleGossipRecord, + database: &NeighborhoodDatabase, + ) -> bool { + let our_key = database.root().public_key(); + debuting_agr.inner.neighbors.iter().any(|pk| pk == our_key) + } + + fn create_second_node_gossip_response( cryptde: &dyn CryptDE, database: &NeighborhoodDatabase, debut_node_key: PublicKey, ) -> Gossip_0v1 { let mut root_node = database.root().clone(); + // Several "second" Nodes may debut before the real second Node is fully established. If + // they do, we want to make sure each of them is isolated from the others and thinks that + // it is the real second Node. root_node.clear_half_neighbors(); root_node .add_half_neighbor_key(debut_node_key.clone()) @@ -395,12 +484,12 @@ impl DebutHandler { } } - fn make_introduction( + fn try_to_make_introduction( &self, database: &NeighborhoodDatabase, debuting_agr: &AccessibleGossipRecord, gossip_source: SocketAddr, - ) -> Option<(Gossip_0v1, PublicKey, NodeAddr)> { + ) -> IntroductionAttempt { if let Some(lcn_key) = Self::find_least_connected_full_neighbor_excluding(database, debuting_agr) { @@ -424,16 +513,16 @@ impl DebutHandler { &debuting_agr.inner.public_key, debut_node_addr ); - Some(( + IntroductionAttempt::Success( GossipBuilder::new(database) .node(database.root().public_key(), true) .node(lcn_key, true) .build(), debuting_agr.inner.public_key.clone(), debut_node_addr, - )) + ) } else { - None + IntroductionAttempt::Failure } } @@ -458,10 +547,8 @@ impl DebutHandler { .full_neighbor_keys(database) .into_iter() .filter(|k| { - database - .node_by_key(*k) - .expect("Node disappeared") - .accepts_connections() + let candidate = database.node_by_key(*k).expect("Node disappeared"); + candidate.accepts_connections() && candidate.routes_data() }) .collect(), database, @@ -515,8 +602,20 @@ impl DebutHandler { keys } - fn should_not_make_another_introduction(debuting_agr: &AccessibleGossipRecord) -> bool { - !debuting_agr.inner.neighbors.is_empty() + fn should_not_make_another_introduction( + db: &NeighborhoodDatabase, + debuting_agr: &AccessibleGossipRecord, + ) -> bool { + let routing_neighbors = + debuting_agr + .inner + .neighbors + .iter() + .filter(|pk| match db.node_by_key(pk) { + Some(node) => node.inner.routes_data, + None => false, + }); + routing_neighbors.count() > 0 } fn make_pass_gossip(database: &NeighborhoodDatabase, pass_target: &PublicKey) -> Gossip_0v1 { @@ -539,7 +638,8 @@ impl NamedType for PassHandler { } impl GossipHandler for PassHandler { - // A Pass must contain a single AGR representing the pass target; it must provide its IP address; + // A Pass must contain a single AGR representing the pass target; the pass target must + // accept connections; it must provide its IP address; // it must specify at least one port; and it must _not_ be sourced by the pass target. fn qualifies( &self, @@ -548,23 +648,23 @@ impl GossipHandler for PassHandler { gossip_source: SocketAddr, ) -> Qualification { if agrs.len() != 1 { - return Qualification::Unmatched; + return Qualification::Unmatched(format!("Pass has 1 Node record, not {}", agrs.len())); } match &agrs[0].node_addr_opt { - None => Qualification::Malformed(format!( - "Pass from {} to {} did not contain NodeAddr", - gossip_source, agrs[0].inner.public_key - )), + None => Qualification::Unmatched("Pass Node record always has a NodeAddr".to_string()), Some(node_addr) => { if node_addr.ports().is_empty() { - Qualification::Malformed(format!( - "Pass from {} to {} at {} contained NodeAddr with no ports", - gossip_source, - agrs[0].inner.public_key, - node_addr.ip_addr() + Qualification::Malformed(Malefactor::new( + Some(agrs[0].inner.public_key.clone()), + Some(gossip_source.ip()), + Some(agrs[0].inner.earning_wallet.clone()), + None, + "Pass contained NodeAddr with no ports".to_string(), )) } else if node_addr.ip_addr() == gossip_source.ip() { - Qualification::Unmatched + Qualification::Unmatched( + "Pass Node record must not be Gossip source".to_string(), + ) } else { Qualification::Matched } @@ -579,7 +679,7 @@ impl GossipHandler for PassHandler { agrs: Vec, _gossip_source: SocketAddr, neighborhood_metadata: NeighborhoodMetadata, - ) -> GossipAcceptanceResult { + ) -> Vec { let pass_agr = &agrs[0]; // empty Gossip shouldn't get here let pass_target_node_addr: NodeAddr = pass_agr .node_addr_opt @@ -608,21 +708,21 @@ impl GossipHandler for PassHandler { }; let mut hash_map = self.previous_pass_targets.borrow_mut(); - let gossip_acceptance_result = match hash_map.get_mut(&pass_target_ip_addr) { + let gossip_acceptance_results = match hash_map.get_mut(&pass_target_ip_addr) { None => match neighborhood_metadata .connection_progress_peers .contains(&pass_target_ip_addr) { true => { send_cpm(ConnectionProgressEvent::PassLoopFound); - GossipAcceptanceResult::Ignored + vec![] } false => { hash_map.insert(pass_target_ip_addr, SystemTime::now()); send_cpm(ConnectionProgressEvent::PassGossipReceived( pass_target_ip_addr, )); - gossip_acceptance_reply() + vec![gossip_acceptance_reply()] } }, Some(timestamp) => { @@ -632,16 +732,20 @@ impl GossipHandler for PassHandler { *timestamp = SystemTime::now(); if duration_since <= PASS_GOSSIP_EXPIRED_TIME { send_cpm(ConnectionProgressEvent::PassLoopFound); - GossipAcceptanceResult::Ignored + vec![] } else { send_cpm(ConnectionProgressEvent::PassGossipReceived( pass_target_ip_addr, )); - gossip_acceptance_reply() + vec![gossip_acceptance_reply()] } } }; - gossip_acceptance_result + gossip_acceptance_results + } + + fn as_any(&self) -> &dyn Any { + self } } @@ -654,6 +758,7 @@ impl PassHandler { } struct IntroductionHandler { + rate_pack_limits: RatePackLimits, logger: Logger, } @@ -666,8 +771,8 @@ impl NamedType for IntroductionHandler { impl GossipHandler for IntroductionHandler { // An Introduction must contain two AGRs, one representing the introducer and one representing // the introducee. Both records must provide their IP addresses. One of the IP addresses must - // match the gossip_source. The other record's IP address must not match the gossip_source. The - // record whose IP address does not match the gossip source must not already be in the database. + // match the gossip_source. The other record's IP address must not match the gossip_source. + // Neither of the two records can be targeted by a half neighborship from this Node. fn qualifies( &self, database: &NeighborhoodDatabase, @@ -683,13 +788,16 @@ impl GossipHandler for IntroductionHandler { Ok(true) => (&agrs[0], &agrs[1]), Ok(false) => (&agrs[1], &agrs[0]), }; - if let Some(qual) = Self::verify_introducer(introducer, database.root()) { + if let Some(qual) = Self::verify_introducer(gossip_source, introducer, database.root()) { return qual; }; if let Some(qual) = Self::verify_introducee(database, introducer, introducee, gossip_source) { return qual; }; + if let Some(qual) = Self::verify_both(database, introducer, introducee) { + return qual; + }; Qualification::Matched } @@ -700,61 +808,87 @@ impl GossipHandler for IntroductionHandler { agrs: Vec, gossip_source: SocketAddr, neighborhood_metadata: NeighborhoodMetadata, - ) -> GossipAcceptanceResult { + ) -> Vec { if database.root().full_neighbor_keys(database).len() >= MAX_DEGREE { - GossipAcceptanceResult::Ignored - } else { - let (introducer, introducee) = Self::identify_players(agrs, gossip_source) - .expect("Introduction not properly qualified"); - let introducer_key = introducer.inner.public_key.clone(); - let introducer_ip_addr = introducer - .node_addr_opt - .as_ref() - .expect("IP Address not found for the Node Addr.") - .ip_addr(); - let introducee_ip_addr = introducee - .node_addr_opt - .as_ref() - .expect("IP Address not found for the Node Addr.") - .ip_addr(); - match self.update_database( - database, + return vec![]; + } + let (introducer, introducee) = Self::identify_players(agrs, gossip_source) + .expect("Introduction not properly qualified"); + let introducer_ban_message_opt = + self.make_introducer_ban_message_if_appropriate(gossip_source, &introducer); + let (introducer_ban_message_opt, introducer_updated) = self + .update_introducer_if_appropriate( cryptde, - introducer, - neighborhood_metadata.user_exit_preferences_opt, - ) { - Ok(_) => (), - Err(e) => { - return GossipAcceptanceResult::Ban(format!( - "Introducer {} tried changing immutable characteristic: {}", - introducer_key, e - )); - } - } + database, + &neighborhood_metadata, + &introducer, + introducer_ban_message_opt, + ); + let introducee_ban_message_opt = + self.make_introducee_ban_message_if_appropriate(&introducee); + let mut results = vec![]; + if let Some(ref introducer_ban_message) = introducer_ban_message_opt { + let ban_result = Self::make_ban_result(&introducer, introducer_ban_message); + results.push(ban_result); + } + if let Some(ref introducee_ban_message) = introducee_ban_message_opt { + let ban_result = Self::make_ban_result(&introducee, introducee_ban_message); + results.push(ban_result); + } + if introducer_ban_message_opt.is_some() { + // If the introducer is banned, we're stopping right here + return results; + } + if introducer_updated { + // If we found out something new about the introducer, gossip it around + results.push(GossipAcceptanceResult::Accepted); + } + if introducee_ban_message_opt.is_none() { + // If the introducee is good, we'll want to debut to it + let (debut, target_key, target_node_addr) = + GossipAcceptorReal::make_debut_triple(database, &introducee) + .expect("Introduction not properly qualified"); + let introducee_ip_addr = target_node_addr.ip_addr(); + results.push(GossipAcceptanceResult::Reply( + debut, + target_key, + target_node_addr, + )); let connection_progress_message = ConnectionProgressMessage { - peer_addr: introducer_ip_addr, + peer_addr: introducer + .node_addr_opt + .as_ref() + .expect("Introducer IP address disappeared") + .ip_addr(), event: ConnectionProgressEvent::IntroductionGossipReceived(introducee_ip_addr), }; neighborhood_metadata .cpm_recipient .try_send(connection_progress_message) .expect("Neighborhood is dead"); - let (debut, target_key, target_node_addr) = - GossipAcceptorReal::make_debut_triple(database, &introducee) - .expect("Introduction not properly qualified"); - GossipAcceptanceResult::Reply(debut, target_key, target_node_addr) } + results + } + + fn as_any(&self) -> &dyn Any { + self } } impl IntroductionHandler { - fn new(logger: Logger) -> IntroductionHandler { - IntroductionHandler { logger } + fn new(rate_pack_limits: &RatePackLimits, logger: Logger) -> IntroductionHandler { + IntroductionHandler { + rate_pack_limits: rate_pack_limits.clone(), + logger, + } } fn verify_size(agrs: &[AccessibleGossipRecord]) -> Option { if agrs.len() != 2 { - return Some(Qualification::Unmatched); + return Some(Qualification::Unmatched(format!( + "Introduction has 2 Node records, not {}", + agrs.len() + ))); } None } @@ -765,12 +899,20 @@ impl IntroductionHandler { ) -> Result { let first_agr = &agrs_ref[0]; let first_ip = match first_agr.node_addr_opt.as_ref() { - None => return Err(Qualification::Unmatched), + None => { + return Err(Qualification::Unmatched( + "Both Node records in Introduction must have NodeAddr".to_string(), + )) + } Some(node_addr) => node_addr.ip_addr(), }; let second_agr = &agrs_ref[1]; let second_ip = match second_agr.node_addr_opt.as_ref() { - None => return Err(Qualification::Unmatched), + None => { + return Err(Qualification::Unmatched( + "Both Node records in Introduction must have NodeAddr".to_string(), + )) + } Some(node_addr) => node_addr.ip_addr(), }; if first_ip == gossip_source.ip() { @@ -778,13 +920,12 @@ impl IntroductionHandler { } else if second_ip == gossip_source.ip() { Ok(false) } else { - Err(Qualification::Malformed(format!( - "In Introduction, neither {} from {} nor {} from {} claims the source IP {}", - first_agr.inner.public_key, - first_ip, - second_agr.inner.public_key, - second_ip, - gossip_source.ip() + Err(Qualification::Malformed(Malefactor::new( + None, + Some(gossip_source.ip()), + None, + None, + "In Introduction, neither Node record claims the source IP".to_string(), ))) } } @@ -807,28 +948,40 @@ impl IntroductionHandler { } fn verify_introducer( - agr: &AccessibleGossipRecord, + gossip_source: SocketAddr, + introducer: &AccessibleGossipRecord, root_node: &NodeRecord, ) -> Option { - if &agr.inner.public_key == root_node.public_key() { - return Some(Qualification::Malformed(format!( - "Introducer {} claims local Node's public key", - agr.inner.public_key + if &introducer.inner.public_key == root_node.public_key() { + return Some(Qualification::Malformed(Malefactor::new( + Some(introducer.inner.public_key.clone()), + Some(gossip_source.ip()), + Some(introducer.inner.earning_wallet.clone()), + None, + "Introducer claims local Node's public key".to_string(), ))); } - let introducer_node_addr = agr.node_addr_opt.as_ref().expect("NodeAddr disappeared"); + let introducer_node_addr = introducer + .node_addr_opt + .as_ref() + .expect("NodeAddr disappeared"); if introducer_node_addr.ports().is_empty() { - return Some(Qualification::Malformed(format!( - "Introducer {} from {} has no ports", - &agr.inner.public_key, - introducer_node_addr.ip_addr() + return Some(Qualification::Malformed(Malefactor::new( + Some(introducer.inner.public_key.clone()), + Some(introducer_node_addr.ip_addr()), + Some(introducer.inner.earning_wallet.clone()), + None, + "Introducer has no ports".to_string(), ))); } if let Some(root_node_addr) = root_node.node_addr_opt() { if introducer_node_addr.ip_addr() == root_node_addr.ip_addr() { - return Some(Qualification::Malformed(format!( - "Introducer {} claims to be at local Node's IP address", - agr.inner.public_key + return Some(Qualification::Malformed(Malefactor::new( + Some(introducer.inner.public_key.clone()), + Some(introducer_node_addr.ip_addr()), + Some(introducer.inner.earning_wallet.clone()), + None, + "Introducer claims to be at local Node's IP address".to_string(), ))); } } @@ -836,28 +989,26 @@ impl IntroductionHandler { } fn verify_introducee( - database: &NeighborhoodDatabase, + _db: &NeighborhoodDatabase, introducer: &AccessibleGossipRecord, introducee: &AccessibleGossipRecord, gossip_source: SocketAddr, ) -> Option { - if database.node_by_key(&introducee.inner.public_key).is_some() { - return Some(Qualification::Unmatched); - } let introducee_node_addr = match introducee.node_addr_opt.as_ref() { - None => return Some(Qualification::Unmatched), + None => { + return Some(Qualification::Unmatched( + "Introducee has no NodeAddr".to_string(), + )) + } Some(node_addr) => node_addr, }; if introducee_node_addr.ports().is_empty() { - return Some(Qualification::Malformed(format!( - "Introducer {} from {} introduced {} from {} with no ports", - &introducer.inner.public_key, - match &introducer.node_addr_opt { - Some(node_addr) => node_addr.ip_addr().to_string(), - None => "?.?.?.?".to_string(), - }, - &introducee.inner.public_key, - introducee_node_addr.ip_addr() + return Some(Qualification::Malformed(Malefactor::new( + Some(introducer.inner.public_key.clone()), + introducer.node_addr_opt.as_ref().map(|na| na.ip_addr()), + Some(introducer.inner.earning_wallet.clone()), + None, + "Introducer introduced introducee with no ports".to_string(), ))); } if introducee @@ -867,16 +1018,34 @@ impl IntroductionHandler { .ip_addr() == gossip_source.ip() { - return Some(Qualification::Malformed(format!( - "Introducer {} and introducee {} both claim {}", - introducer.inner.public_key, - introducee.inner.public_key, - gossip_source.ip() + return Some(Qualification::Malformed(Malefactor::new( + Some(introducer.inner.public_key.clone()), + Some(gossip_source.ip()), + Some(introducer.inner.earning_wallet.clone()), + None, + "Introducer claims the same IP address as its introducee".to_string(), ))); } None } + fn verify_both( + db: &NeighborhoodDatabase, + introducer: &AccessibleGossipRecord, + introducee: &AccessibleGossipRecord, + ) -> Option { + let half_neighbors = db.root().half_neighbor_keys(); + if half_neighbors.contains(&introducee.inner.public_key) + && half_neighbors.contains(&introducer.inner.public_key) + { + Some(Qualification::Unmatched( + "Introducer and introducee are already our half-neighbors".to_string(), + )) + } else { + None + } + } + fn update_database( &self, database: &mut NeighborhoodDatabase, @@ -932,9 +1101,97 @@ impl IntroductionHandler { trace!(self.logger, "Current database: {}", database.to_dot_graph()); Ok(true) } + + fn make_introducer_ban_message_if_appropriate( + &self, + gossip_source: SocketAddr, + introducer: &AccessibleGossipRecord, + ) -> Option { + GossipAcceptorReal::validate_incoming_agr( + introducer, + format!( + "Introducer {} from {}", + introducer.inner.public_key, gossip_source + ), + &self.rate_pack_limits, + &self.logger, + ) + .err() + } + + fn make_introducee_ban_message_if_appropriate( + &self, + introducee: &AccessibleGossipRecord, + ) -> Option { + GossipAcceptorReal::validate_incoming_agr( + introducee, + format!( + "Introducee {} at {}", + introducee.inner.public_key, + introducee + .node_addr_opt + .as_ref() + .expect("NodeAddr disappeared") + .ip_addr(), + ), + &self.rate_pack_limits, + &self.logger, + ) + .err() + } + + fn update_introducer_if_appropriate( + &self, + cryptde: &dyn CryptDE, + database: &mut NeighborhoodDatabase, + neighborhood_metadata: &NeighborhoodMetadata, + introducer: &AccessibleGossipRecord, + introducer_ban_message_opt: Option, + ) -> (Option, bool) { + if introducer_ban_message_opt.is_none() { + match self.update_database( + database, + cryptde, + introducer.clone(), + neighborhood_metadata.user_exit_preferences_opt.clone(), + ) { + Ok(updated) => (None, updated), + Err(e) => { + let message = format!( + "Introducer {} tried changing immutable characteristic: {}", + introducer.inner.public_key, e + ); + (Some(message), false) + } + } + } else { + (introducer_ban_message_opt, false) + } + } + + fn make_ban_result( + agr: &AccessibleGossipRecord, + agr_ban_message: &str, + ) -> GossipAcceptanceResult { + let agr_key = agr.inner.public_key.clone(); + let agr_wallet = agr.inner.earning_wallet.clone(); + let agr_ip_address = agr + .node_addr_opt + .as_ref() + .expect("AGR IP address disappeared") + .ip_addr(); + GossipAcceptanceResult::Ban(Malefactor::new( + Some(agr_key), + Some(agr_ip_address), + Some(agr_wallet), + None, + agr_ban_message.to_string(), + )) + } } struct StandardGossipHandler { + rate_pack_limits: RatePackLimits, logger: Logger, } @@ -947,6 +1204,7 @@ impl NamedType for StandardGossipHandler { impl GossipHandler for StandardGossipHandler { // Standard Gossip must not be a Debut, Pass, or Introduction. There must be no record in the // Gossip describing the local Node (although there may be records that reference the local Node as a neighbor). + // There must be no Node in the Gossip that claims to reside at this Node's IP address. fn qualifies( &self, database: &NeighborhoodDatabase, @@ -954,37 +1212,49 @@ impl GossipHandler for StandardGossipHandler { gossip_source: SocketAddr, ) -> Qualification { // must-not-be-debut-pass-or-introduction is assured by StandardGossipHandler's placement in the gossip_handlers list - let agrs_next_door = agrs + // Check to make sure no record claims this Node's IP address + let agrs_with_ips = agrs .iter() .filter(|agr| agr.node_addr_opt.is_some()) .collect::>(); let root_node = database.root(); if root_node.accepts_connections() { - if let Some(impostor) = agrs_next_door.iter().find(|agr| { - Self::ip_of(agr) + if let Some(impostor) = agrs_with_ips.iter().find(|agr_with_ip| { + Self::ip_of(agr_with_ip) == root_node .node_addr_opt() .expect("Root Node that accepts connections must have NodeAddr") .ip_addr() }) { - return Qualification::Malformed( - format!("Standard Gossip from {} contains a record claiming that {} has this Node's IP address", - gossip_source, - impostor.inner.public_key)); + return Qualification::Malformed(Malefactor::new( + None, + Some(gossip_source.ip()), + None, + None, + format!( + "Standard Gossip contains a record claiming that {} resides at this Node's IP address", + impostor.inner.public_key + ) + )); } } - if agrs + // Check to make sure no record claims this Node's public key + if let Some(impostor) = agrs .iter() - .any(|agr| &agr.inner.public_key == root_node.public_key()) + .find(|agr| &agr.inner.public_key == root_node.public_key()) { - return Qualification::Malformed(format!( - "Standard Gossip from {} contains a record with this Node's public key", - gossip_source + return Qualification::Malformed(Malefactor::new( + Some(impostor.inner.public_key.clone()), + None, + Some(impostor.inner.earning_wallet.clone()), + None, + "Standard Gossip contains a record claiming this Node's public key".to_string(), )); } + // Check for duplicate IP addresses in the Gossip let init_addr_set: HashSet = HashSet::new(); let init_dup_set: HashSet = HashSet::new(); - let dup_set = agrs_next_door + let dup_set = agrs_with_ips .into_iter() .fold((init_addr_set, init_dup_set), |so_far, agr| { let (addr_set, dup_set) = so_far; @@ -996,15 +1266,20 @@ impl GossipHandler for StandardGossipHandler { } }) .1; - if dup_set.is_empty() { Qualification::Matched } else { let dup_vec = dup_set.into_iter().take(1).collect::>(); let first_dup_ip = dup_vec.first().expect("Duplicate IP address disappeared"); - Qualification::Malformed(format!( - "Standard Gossip from {} contains multiple records claiming to be from {}", - gossip_source, first_dup_ip + Qualification::Malformed(Malefactor::new( + None, + Some(gossip_source.ip()), + None, + None, + format!( + "Standard Gossip contains multiple records claiming to be from {}", + first_dup_ip + ), )) } } @@ -1016,27 +1291,72 @@ impl GossipHandler for StandardGossipHandler { agrs: Vec, gossip_source: SocketAddr, neighborhood_metadata: NeighborhoodMetadata, - ) -> GossipAcceptanceResult { + ) -> Vec { + // TODO: + // We should look up the source Node in the database by gossip_source and validate all + // its immutable data (public key, earning wallet, maybe others). If any of that has changed, + // another Malefactor ban. let initial_neighborship_status = StandardGossipHandler::check_full_neighbor(database, gossip_source.ip()); - + let gossip_source_agr = match agrs.iter().find(|agr| { + if agr.inner.accepts_connections { + // If it accepts connections, it'll know its IP address, and we can match on gossip_source + agr.node_addr_opt.as_ref().map(|na| na.ip_addr()) == Some(gossip_source.ip()) + } else { + // If it doesn't, we'll know its IP address but it won't, so we'll look it up by IP + // and check public keys. + if let Some(db_node) = database.node_by_ip(&gossip_source.ip()) { + db_node.public_key() == &agr.inner.public_key + } else { + false + } + } + }) { + Some(agr) => agr.clone(), + None => { + let message = format!( + "Node at {} sent Standard gossip without a record describing itself", + gossip_source.ip() + ); + warning!(self.logger, "{}", message); + return vec![GossipAcceptanceResult::Ban(Malefactor::new( + None, + Some(gossip_source.ip()), + None, + None, + message, + ))]; + } + }; let patch = self.compute_patch(&agrs, database.root(), neighborhood_metadata.db_patch_size); - let filtered_agrs = self.filter_agrs_by_patch(agrs, patch); + let in_patch_agrs = agrs + .into_iter() + .filter(|agr| self.contained_by_patch(agr, &patch)) + .collect_vec(); + + let (worthy_agrs, malefactor_bans) = + self.extract_malefactors(in_patch_agrs, database, &gossip_source_agr, gossip_source); + let mut db_changed = false; + + let (new_agrs, obsolete_agrs) = + self.identify_non_introductory_new_and_obsolete_nodes(database, worthy_agrs); - let mut db_changed = self.identify_and_add_non_introductory_new_nodes( + db_changed |= !new_agrs.is_empty(); + self.add_new_nodes( database, - &filtered_agrs, - gossip_source, + new_agrs, neighborhood_metadata.user_exit_preferences_opt.as_ref(), ); - db_changed = self.identify_and_update_obsolete_nodes(database, filtered_agrs) || db_changed; - db_changed = - self.add_src_node_as_half_neighbor(cryptde, database, gossip_source) || db_changed; + + db_changed |= !obsolete_agrs.is_empty(); + self.update_obsolete_nodes(database, obsolete_agrs); + + db_changed |= self.add_src_node_as_half_neighbor(cryptde, database, gossip_source); let final_neighborship_status = StandardGossipHandler::check_full_neighbor(database, gossip_source.ip()); // If no Nodes need updating, return ::Ignored and don't change the database. // Otherwise, return ::Accepted. - if db_changed { + let mut response = if db_changed { trace!(self.logger, "Current database: {}", database.to_dot_graph()); if (initial_neighborship_status, final_neighborship_status) == (false, true) { // Received Reply for Acceptance of Debut Gossip (false, true) @@ -1049,20 +1369,29 @@ impl GossipHandler for StandardGossipHandler { .try_send(cpm) .unwrap_or_else(|e| panic!("Neighborhood is dead: {}", e)); } - GossipAcceptanceResult::Accepted + vec![GossipAcceptanceResult::Accepted] } else { debug!( self.logger, "Gossip contained nothing new: StandardGossipHandler is ignoring it" ); - GossipAcceptanceResult::Ignored - } + vec![] + }; + response.extend(malefactor_bans); + response + } + + fn as_any(&self) -> &dyn Any { + self } } impl StandardGossipHandler { - fn new(logger: Logger) -> StandardGossipHandler { - StandardGossipHandler { logger } + fn new(rate_pack_limits: &RatePackLimits, logger: Logger) -> StandardGossipHandler { + StandardGossipHandler { + rate_pack_limits: rate_pack_limits.clone(), + logger, + } } fn compute_patch( @@ -1109,7 +1438,7 @@ impl StandardGossipHandler { patch.remove(current_node_key); trace!( self.logger, - "While computing patch no AGR record found for public key {:?}", + "While computing patch no Node record found for public key {:?}", current_node_key ); return; @@ -1124,78 +1453,168 @@ impl StandardGossipHandler { } } - fn filter_agrs_by_patch( + fn contained_by_patch(&self, agr: &AccessibleGossipRecord, patch: &HashSet) -> bool { + patch.contains(&agr.inner.public_key) + } + + fn extract_malefactors( &self, agrs: Vec, - patch: HashSet, - ) -> Vec { - agrs.into_iter() - .filter(|agr| patch.contains(&agr.inner.public_key)) - .collect::>() + database: &NeighborhoodDatabase, + gossip_source_agr: &AccessibleGossipRecord, + // If our gossip_source_agr doesn't accept connections, it won't have an IP address; so + // we accept that separately. + gossip_source: SocketAddr, + ) -> (Vec, Vec) { + let gossip_source_ip = gossip_source.ip(); + // TODO: This would be more consistent with identify_non_introductory_new_and_obsolete_nodes + // below if it used a for loop with mutation. + let (valid_agrs, bans) = agrs.into_iter().fold((vec![], vec![]), |so_far, agr| { + let (mut valid_agrs, mut bans) = so_far; + match GossipAcceptorReal::validate_incoming_agr( + &agr, + format!( + "Node {} from Standard gossip received from {}", + agr.inner.public_key, gossip_source_ip, + ), + &self.rate_pack_limits, + &self.logger, + ) { + Ok(_) => valid_agrs.push(agr), + Err(ban_message) => { + bans.push(GossipAcceptanceResult::Ban(Malefactor::new( + Some(agr.inner.public_key.clone()), + agr.node_addr_opt.map(|na| na.ip_addr()), + Some(agr.inner.earning_wallet.clone()), + None, + ban_message, + ))); + } + } + (valid_agrs, bans) + }); + let next_door_neighbor_keys = database + .root() + .inner + .neighbors + .iter() + .collect::>(); + let (valid_agrs, bans) = + valid_agrs.into_iter().fold((vec![], bans), |so_far, agr| { + let (mut valid_agrs, mut bans) = so_far; + if &agr.inner.public_key == database.root_key() { + let ip_addr_opt = agr.node_addr_opt.map(|na| na.ip_addr()); + bans.push(GossipAcceptanceResult::Ban(Malefactor::new( + Some(agr.inner.public_key.clone()), + ip_addr_opt, + Some(agr.inner.earning_wallet.clone()), + None, + format!( + "Node {} at {} sent Standard gossip that contained a record claiming our own public key", + agr.inner.public_key, + match ip_addr_opt { + Some(ip) => ip.to_string(), + None => "?.?.?.?".to_string(), + } + ) + ))); + } + else if + // the AGR has an IP address + agr.node_addr_opt.is_some() && + // the AGR's IP address doesn't match the gossip source's IP address + (agr.node_addr_opt.as_ref().expect("NodeAddr disappeared").ip_addr() != gossip_source_ip) && + // the AGR's public key isn't a next-door neighbor + !next_door_neighbor_keys.contains(&&agr.inner.public_key) + { + bans.push(GossipAcceptanceResult::Ban(Malefactor::new( + Some(gossip_source_agr.inner.public_key.clone()), + Some(gossip_source_ip), + Some(gossip_source_agr.inner.earning_wallet.clone()), + None, + format!( + "Node {} at {:?} sent Standard gossip that contained an IP address for victim Node {} that we should not have known", + gossip_source_agr.inner.public_key, + gossip_source_ip, + agr.inner.public_key, + ), + ))); + } + else { + valid_agrs.push(agr) + } + (valid_agrs, bans) + }); + (valid_agrs, bans) } - fn identify_and_add_non_introductory_new_nodes( + fn identify_non_introductory_new_and_obsolete_nodes( &self, database: &mut NeighborhoodDatabase, - agrs: &[AccessibleGossipRecord], - gossip_source: SocketAddr, - user_exit_preferences_opt: Option<&UserExitPreferences>, - ) -> bool { + agrs: Vec, + ) -> (Vec, Vec) { let all_keys = database .keys() .into_iter() .cloned() .collect::>(); - agrs.iter() - .filter(|agr| !all_keys.contains(&agr.inner.public_key)) - // TODO: A node that tells us the IP Address of the node that isn't in our database should be malefactor banned - .filter(|agr| match &agr.node_addr_opt { - None => true, - Some(node_addr) => { - let socket_addrs: Vec = node_addr.clone().into(); - socket_addrs.contains(&gossip_source) - } - }) - .for_each(|agr| { - let mut node_record = NodeRecord::from(agr); - match user_exit_preferences_opt { - Some(user_exit_preferences) => { - user_exit_preferences.assign_nodes_country_undesirability(&mut node_record) - } - None => (), + let mut new_nodes = vec![]; + let mut obsolete_nodes = vec![]; + for agr in agrs { + if !all_keys.contains(&agr.inner.public_key) { + new_nodes.push(agr); + } else if let Some(existing_node) = database.node_by_key(&agr.inner.public_key) { + if agr.inner.version > existing_node.version() { + obsolete_nodes.push(agr); } - trace!( - self.logger, - "Discovered new Node {:?}: {:?}", - node_record.public_key(), - node_record.full_neighbor_keys(database) - ); - database - .add_node(node_record) - .expect("List of new Nodes contained existing Nodes"); - }); - database.keys().len() != all_keys.len() + } + } + (new_nodes, obsolete_nodes) } - fn identify_and_update_obsolete_nodes( + fn add_new_nodes( &self, database: &mut NeighborhoodDatabase, agrs: Vec, - ) -> bool { - agrs.into_iter().fold(false, |b, agr| { - match database.node_by_key(&agr.inner.public_key) { - Some(existing_node) if agr.inner.version > existing_node.version() => { - trace!( - self.logger, - "Updating Node {:?} from v{} to v{}", - existing_node.public_key(), - existing_node.version(), - agr.inner.version - ); - self.update_database_record(database, agr) || b + user_exit_preferences_opt: Option<&UserExitPreferences>, + ) { + agrs.into_iter().for_each(|agr| { + let mut node_record = NodeRecord::from(agr); + match user_exit_preferences_opt { + Some(user_exit_preferences) => { + user_exit_preferences.assign_nodes_country_undesirability(&mut node_record) } - _ => b, + None => (), } + trace!( + self.logger, + "Discovered new Node {:?}: {:?}", + node_record.public_key(), + node_record.full_neighbor_keys(database) + ); + database + .add_node(node_record) + .expect("List of new Nodes contained existing Nodes"); + }); + } + + fn update_obsolete_nodes( + &self, + database: &mut NeighborhoodDatabase, + agrs: Vec, + ) { + agrs.into_iter().for_each(|agr| { + let existing_node = database + .node_by_key(&agr.inner.public_key) + .expect("Node magically disappeared from neighborhood database"); + trace!( + self.logger, + "Updating Node {:?} from v{} to v{}", + existing_node.public_key(), + existing_node.version(), + agr.inner.version + ); + self.update_database_record(database, agr); }) } @@ -1287,10 +1706,15 @@ impl GossipHandler for RejectHandler { agrs: &[AccessibleGossipRecord], gossip_source: SocketAddr, ) -> Qualification { - Qualification::Malformed(format!( - "Gossip with {} records from {} is unclassifiable by any qualifier", - agrs.len(), - gossip_source + Qualification::Malformed(Malefactor::new( + None, + Some(gossip_source.ip()), + None, + None, + format!( + "Gossip with {} records is unclassifiable by any qualifier", + agrs.len(), + ), )) } @@ -1301,9 +1725,13 @@ impl GossipHandler for RejectHandler { _agrs: Vec, _gossip_source: SocketAddr, _neighborhood_metadata: NeighborhoodMetadata, - ) -> GossipAcceptanceResult { + ) -> Vec { panic!("Should never be called") } + + fn as_any(&self) -> &dyn Any { + self + } } impl RejectHandler { @@ -1319,7 +1747,7 @@ pub trait GossipAcceptor: Send /* Send because lazily-written tests require it * agrs: Vec, gossip_source: SocketAddr, neighborhood_metadata: NeighborhoodMetadata, - ) -> GossipAcceptanceResult; + ) -> Vec; } pub struct GossipAcceptorReal { @@ -1335,12 +1763,24 @@ impl GossipAcceptor for GossipAcceptorReal { agrs: Vec, gossip_source: SocketAddr, neighborhood_metadata: NeighborhoodMetadata, - ) -> GossipAcceptanceResult { + ) -> Vec { let (qualification, handler_ref) = self .gossip_handlers .iter() .map(|h| (h.qualifies(database, &agrs, gossip_source), h.as_ref())) - .find(|pair| !matches!(pair, (Qualification::Unmatched, _))) + .map(|(qualification, handler_ref)| { + if let Qualification::Unmatched(msg) = &qualification { + trace!( + self.logger, + "Gossip from {} did not qualify for {}: {}", + gossip_source, + handler_ref.type_name(), + msg + ); + } + (qualification, handler_ref) + }) + .find(|pair| !matches!(pair, (Qualification::Unmatched(_), _))) .expect("gossip_handlers should intercept everything"); match qualification { Qualification::Matched => { @@ -1357,23 +1797,32 @@ impl GossipAcceptor for GossipAcceptorReal { neighborhood_metadata, ) } - Qualification::Unmatched => { + Qualification::Unmatched(_) => { panic!("Nothing in gossip_handlers returned Matched or Malformed") } - Qualification::Malformed(reason) => GossipAcceptanceResult::Ban(reason), + Qualification::Malformed(malefactor) => { + vec![GossipAcceptanceResult::Ban(malefactor)] + } } } } impl GossipAcceptorReal { - pub fn new(cryptde: Box) -> GossipAcceptorReal { + pub fn new( + cryptde: Box, + persistent_config: &dyn PersistentConfiguration, + ) -> GossipAcceptorReal { + let rate_pack_limits = &persistent_config + .rate_pack_limits() + .expect("RatePackLimits should be set"); + let logger = Logger::new("GossipAcceptor"); GossipAcceptorReal { gossip_handlers: vec![ - Box::new(DebutHandler::new(logger.clone())), + Box::new(DebutHandler::new(rate_pack_limits, logger.clone())), Box::new(PassHandler::new()), - Box::new(IntroductionHandler::new(logger.clone())), - Box::new(StandardGossipHandler::new(logger.clone())), + Box::new(IntroductionHandler::new(rate_pack_limits, logger.clone())), + Box::new(StandardGossipHandler::new(rate_pack_limits, logger.clone())), Box::new(RejectHandler::new()), ], cryptde, @@ -1412,6 +1861,58 @@ impl GossipAcceptorReal { debut_target_node_addr.clone(), )) } + + fn validate_incoming_agr( + agr: &AccessibleGossipRecord, + agr_description: String, + rate_pack_limits: &RatePackLimits, + logger: &Logger, + ) -> Result<(), String> { + if agr.inner.routes_data { + return match rate_pack_limits.analyze(&agr.inner.rate_pack) { + Ok(_) => Ok(()), + Err(e) => { + let message = format!( + "{} rejected due to rate pack limit violation: {:?}", + agr_description, e + ); + warning!(logger, "{}", message.as_str()); + Err(message) + } + }; + } + Ok(()) + } +} + +pub struct GossipAcceptorInvalid {} + +impl GossipAcceptor for GossipAcceptorInvalid { + fn handle( + &self, + _database: &mut NeighborhoodDatabase, + _agrs: Vec, + _gossip_source: SocketAddr, + _neighborhood_metadata: NeighborhoodMetadata, + ) -> Vec { + Self::invalid() + } +} + +impl GossipAcceptorInvalid { + pub fn new() -> Self { + Self {} + } + + fn invalid() -> ! { + panic!("GossipAcceptor was never initialized"); + } +} + +impl Default for GossipAcceptorInvalid { + fn default() -> Self { + Self::new() + } } #[cfg(test)] @@ -1426,13 +1927,18 @@ mod tests { UNREACHABLE_COUNTRY_PENALTY, }; use crate::sub_lib::cryptde_null::CryptDENull; - use crate::sub_lib::neighborhood::{ConnectionProgressEvent, ConnectionProgressMessage}; + use crate::sub_lib::neighborhood::{ + ConnectionProgressEvent, ConnectionProgressMessage, RatePack, DEFAULT_RATE_PACK, + DEFAULT_RATE_PACK_LIMITS, ZERO_RATE_PACK, + }; use crate::sub_lib::utils::time_t_timestamp; + use crate::sub_lib::wallet::Wallet; use crate::test_utils::neighborhood_test_utils::{ db_from_node, gossip_about_nodes_from_database, linearly_connect_nodes, make_meaningless_db, make_node_record, make_node_record_cc, make_node_record_f, make_node_records, public_keys_from_node_records, DB_PATCH_SIZE_FOR_TEST, }; + use crate::test_utils::persistent_configuration_mock::PersistentConfigurationMock; use crate::test_utils::unshared_test_utils::make_cpm_recipient; use crate::test_utils::{assert_contains, vec_to_set}; use actix::System; @@ -1442,6 +1948,7 @@ mod tests { use masq_lib::messages::ExitLocation; use masq_lib::test_utils::logging::{init_test_logging, TestLogHandler}; use masq_lib::test_utils::utils::TEST_DEFAULT_CHAIN; + use masq_lib::utils::NeighborhoodModeLight; use std::convert::TryInto; use std::net::Ipv4Addr; use std::ops::{Add, Sub}; @@ -1449,21 +1956,35 @@ mod tests { use std::time::Duration; lazy_static! { - static ref CRYPTDE_PAIR: CryptDEPair = CryptDEPair::null(); + static ref GA_CRYPTDE_PAIR: CryptDEPair = CryptDEPair::null(); } - #[test] - fn constants_have_correct_values() { - assert_eq!(MAX_DEGREE, 5); - assert_eq!(PASS_GOSSIP_EXPIRED_TIME, Duration::from_secs(60)); + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + struct Mode { + pub accepts_connections: bool, + pub routes_data: bool, } - #[derive(Clone, Copy, Debug, PartialEq, Eq)] - enum Mode { - Standard, - OriginateOnly, - // GossipAcceptor doesn't care about ConsumeOnly; that's routing, not Gossip - // ZeroHop is decentralized and should never appear in GossipAcceptor tests + impl Mode { + pub fn new(accepts_connections: bool, routes_data: bool) -> Self { + Self { + accepts_connections, + routes_data, + } + } + } + + impl From for Mode { + fn from(neighborhood_mode: NeighborhoodModeLight) -> Self { + match neighborhood_mode { + NeighborhoodModeLight::Standard => Mode::new(true, true), + NeighborhoodModeLight::OriginateOnly => Mode::new(false, true), + NeighborhoodModeLight::ConsumeOnly => Mode::new(false, false), + NeighborhoodModeLight::ZeroHop => { + unimplemented!("ZeroHop should not be used in GossipAcceptor tests") + } + } + } } fn make_default_neighborhood_metadata() -> NeighborhoodMetadata { @@ -1475,16 +1996,84 @@ mod tests { } } + impl RatePackLimits { + pub fn test_default() -> RatePackLimits { + Self { + lo: RatePack::new(u64::MIN, u64::MIN, u64::MIN, u64::MIN), + hi: RatePack::new(u64::MAX, u64::MAX, u64::MAX, u64::MAX), + } + } + } + + #[test] + fn constants_have_correct_values() { + assert_eq!(MAX_DEGREE, 5); + assert_eq!(PASS_GOSSIP_EXPIRED_TIME, Duration::from_secs(60)); + } + + #[test] + fn gossip_acceptor_constructor_creates_handlers_properly() { + let subject = GossipAcceptorReal::new( + GA_CRYPTDE_PAIR.main.dup(), + &PersistentConfigurationMock::new() + .rate_pack_limits_result(Ok(RatePackLimits::test_default())), + ); + + let debut_handler: &DebutHandler = subject.gossip_handlers[0] + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + debut_handler.rate_pack_limits, + RatePackLimits::test_default() + ); + assert_eq!(debut_handler.logger.name(), "GossipAcceptor"); + + let _: &PassHandler = subject.gossip_handlers[1] + .as_any() + .downcast_ref::() + .unwrap(); + // The fact that the downcast didn't panic is assertion enough + + let introduction_handler: &IntroductionHandler = subject.gossip_handlers[2] + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + introduction_handler.rate_pack_limits, + RatePackLimits::test_default() + ); + assert_eq!(introduction_handler.logger.name(), "GossipAcceptor"); + + let standard_gossip_handler: &StandardGossipHandler = subject.gossip_handlers[3] + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + standard_gossip_handler.rate_pack_limits, + RatePackLimits::test_default() + ); + assert_eq!(standard_gossip_handler.logger.name(), "GossipAcceptor"); + + let _: &RejectHandler = subject.gossip_handlers[4] + .as_any() + .downcast_ref::() + .unwrap(); + // The fact that the downcast didn't panic is assertion enough + assert_eq!(subject.gossip_handlers.len(), 5); + } + #[test] fn proper_debut_of_accepting_node_with_populated_database_is_identified_and_handled() { - let (gossip, new_node, gossip_source_opt) = make_debut(2345, Mode::Standard); + let (gossip, new_node, gossip_source_opt) = + make_debut(2345, NeighborhoodModeLight::Standard.into()); let root_node = make_node_record(1234, true); let mut db = db_from_node(&root_node); let neighbor_key = &db.add_node(make_node_record(3456, true)).unwrap(); db.add_arbitrary_full_neighbor(root_node.public_key(), neighbor_key); let cryptde = CryptDENull::from(db.root().public_key(), TEST_DEFAULT_CHAIN); let agrs_vec: Vec = gossip.try_into().unwrap(); - let subject = DebutHandler::new(Logger::new("test")); + let subject = DebutHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let qualifies_result = subject.qualifies(&db, &agrs_vec.as_slice(), gossip_source_opt.clone()); @@ -1503,88 +2092,56 @@ mod tests { .build(); assert_eq!( handle_result, - GossipAcceptanceResult::Reply( + vec![GossipAcceptanceResult::Reply( introduction, new_node.public_key().clone(), new_node.node_addr_opt().unwrap(), - ), + )], ); } #[test] fn proper_debut_of_non_accepting_node_with_populated_database_is_identified_and_handled() { - let (gossip, new_node, gossip_source) = make_debut(2345, Mode::OriginateOnly); + let (gossip, new_node, gossip_source) = + make_debut(2345, NeighborhoodModeLight::OriginateOnly.into()); let root_node = make_node_record(1234, true); - let mut db = db_from_node(&root_node); - let neighbor_key = &db.add_node(make_node_record(3456, true)).unwrap(); - db.add_arbitrary_full_neighbor(root_node.public_key(), neighbor_key); - let cryptde = CryptDENull::from(db.root().public_key(), TEST_DEFAULT_CHAIN); + let mut dest_db = db_from_node(&root_node); + let neighbor_key = &dest_db.add_node(make_node_record(3456, true)).unwrap(); + dest_db.add_arbitrary_full_neighbor(root_node.public_key(), neighbor_key); + let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); let agrs_vec: Vec = gossip.try_into().unwrap(); - let subject = DebutHandler::new(Logger::new("test")); + let subject = DebutHandler::new(&RatePackLimits::test_default(), Logger::new("test")); - let qualifies_result = subject.qualifies(&db, agrs_vec.as_slice(), gossip_source.clone()); + let qualifies_result = + subject.qualifies(&dest_db, agrs_vec.as_slice(), gossip_source.clone()); let handle_result = subject.handle( &cryptde, - &mut db, + &mut dest_db, agrs_vec, gossip_source, make_default_neighborhood_metadata(), ); assert_eq!(Qualification::Matched, qualifies_result); - let introduction = GossipBuilder::new(&db) - .node(db.root().public_key(), true) + let introduction = GossipBuilder::new(&dest_db) + .node(dest_db.root().public_key(), true) .node(neighbor_key, true) .build(); assert_eq!( handle_result, - GossipAcceptanceResult::Reply( + vec![GossipAcceptanceResult::Reply( introduction, new_node.public_key().clone(), NodeAddr::from(&gossip_source), - ), + )], ); - } - - #[test] - fn proper_debut_of_node_cant_produce_introduction_because_of_common_neighbor() { - let src_root = make_node_record(1234, true); - let mut src_db = db_from_node(&src_root); - let cryptde = CryptDENull::from(src_db.root().public_key(), TEST_DEFAULT_CHAIN); - let dest_root = make_node_record(2345, true); - let mut dest_db = db_from_node(&dest_root); - let one_common_neighbor = make_node_record(3456, true); - let another_common_neighbor = make_node_record(4567, true); - src_db.add_node(one_common_neighbor.clone()).unwrap(); - src_db.add_arbitrary_full_neighbor(src_root.public_key(), one_common_neighbor.public_key()); - src_db.add_node(another_common_neighbor.clone()).unwrap(); - src_db.add_arbitrary_full_neighbor( - src_root.public_key(), - another_common_neighbor.public_key(), - ); - dest_db.add_node(one_common_neighbor.clone()).unwrap(); - dest_db - .add_arbitrary_full_neighbor(dest_root.public_key(), one_common_neighbor.public_key()); - dest_db.add_node(another_common_neighbor.clone()).unwrap(); - dest_db.add_arbitrary_full_neighbor( - dest_root.public_key(), - another_common_neighbor.public_key(), - ); - let gossip = GossipBuilder::new(&src_db) - .node(src_db.root().public_key(), true) - .build(); - let agrs_vec: Vec = gossip.try_into().unwrap(); - let subject = DebutHandler::new(Logger::new("test")); - - let result = subject.handle( - &cryptde, - &mut dest_db, - agrs_vec, - src_root.node_addr_opt().unwrap().into(), - make_default_neighborhood_metadata(), + // Version of debuting Node in destination database should have NodeAddr + // even though it doesn't accept connections + let debuted_node = dest_db.node_by_key(&new_node.public_key()).unwrap(); + assert_eq!( + debuted_node.metadata.node_addr_opt, + Some(NodeAddr::from(&gossip_source)) ); - - assert_eq!(result, GossipAcceptanceResult::Accepted); } #[test] @@ -1606,7 +2163,7 @@ mod tests { .node(src_db.root().public_key(), true) .build(); let agrs_vec: Vec = gossip.try_into().unwrap(); - let subject = DebutHandler::new(Logger::new("test")); + let subject = DebutHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let result = subject.handle( &cryptde, @@ -1618,18 +2175,19 @@ mod tests { assert_eq!( result, - GossipAcceptanceResult::Failed( + vec![GossipAcceptanceResult::Failed( GossipFailure_0v1::NoSuitableNeighbors, src_root.public_key().clone(), src_root.node_addr_opt().unwrap(), - ) + )] ); } #[test] fn debut_with_node_addr_not_accepting_connections_is_rejected() { - let (mut gossip, _j, gossip_source) = make_debut(2345, Mode::OriginateOnly); - let subject = DebutHandler::new(Logger::new("test")); + let (mut gossip, _j, gossip_source) = + make_debut(2345, NeighborhoodModeLight::OriginateOnly.into()); + let subject = DebutHandler::new(&RatePackLimits::test_default(), Logger::new("test")); gossip.node_records[0].node_addr_opt = Some(NodeAddr::new( &IpAddr::from_str("1.2.3.4").unwrap(), &[1234], @@ -1640,65 +2198,83 @@ mod tests { assert_eq!( result, - Qualification::Malformed( - "Debut from 200.200.200.200:2000 for AgMEBQ does not accept connections, yet contained NodeAddr".to_string() - ), + Qualification::Malformed(Malefactor::new( + Some(agrs_vec[0].inner.public_key.clone()), + Some(gossip_source.ip()), + Some(agrs_vec[0].inner.earning_wallet.clone()), + None, + "Debut does not accept connections, yet contained NodeAddr".to_string() + )), ); } #[test] fn debut_without_node_addr_accepting_connections_is_rejected() { - let (mut gossip, _j, gossip_source) = make_debut(2345, Mode::Standard); + let (mut gossip, _j, gossip_source) = + make_debut(2345, NeighborhoodModeLight::Standard.into()); gossip.node_records[0].node_addr_opt = None; - let subject = DebutHandler::new(Logger::new("test")); + let subject = DebutHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs_vec: Vec = gossip.try_into().unwrap(); let result = subject.qualifies(&make_meaningless_db(), agrs_vec.as_slice(), gossip_source); assert_eq!( result, - Qualification::Malformed( - "Debut from 2.3.4.5:2345 for AgMEBQ contained no NodeAddr".to_string() - ), + Qualification::Malformed(Malefactor::new( + Some(agrs_vec[0].inner.public_key.clone()), + Some(gossip_source.ip()), + Some(agrs_vec[0].inner.earning_wallet.clone()), + None, + "Debut contained no NodeAddr".to_string() + )), ); } #[test] fn debut_without_node_addr_ports_accepting_connections_is_rejected() { - let (mut gossip, _, gossip_source) = make_debut(2345, Mode::Standard); + let (mut gossip, _, gossip_source) = + make_debut(2345, NeighborhoodModeLight::Standard.into()); gossip.node_records[0].node_addr_opt = Some(NodeAddr::new(&IpAddr::from_str("1.2.3.4").unwrap(), &[])); - let subject = DebutHandler::new(Logger::new("test")); + let subject = DebutHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs_vec: Vec = gossip.try_into().unwrap(); let result = subject.qualifies(&make_meaningless_db(), agrs_vec.as_slice(), gossip_source); assert_eq!( result, - Qualification::Malformed( - "Debut from 2.3.4.5:2345 for AgMEBQ contained NodeAddr with no ports".to_string() - ), + Qualification::Malformed(Malefactor::new( + Some(agrs_vec[0].inner.public_key.clone()), + Some(gossip_source.ip()), + Some(agrs_vec[0].inner.earning_wallet.clone()), + None, + "Debut contained NodeAddr with no ports".to_string() + )), ); } #[test] fn apparent_debut_with_node_already_in_database_is_unmatched() { - let (gossip, new_node, gossip_source) = make_debut(2345, Mode::Standard); + let (gossip, new_node, gossip_source) = + make_debut(2345, NeighborhoodModeLight::Standard.into()); let root_node = make_node_record(1234, true); let mut db = db_from_node(&root_node); let neighbor_key = &db.add_node(make_node_record(3456, true)).unwrap(); db.add_arbitrary_full_neighbor(root_node.public_key(), neighbor_key); db.add_node(new_node.clone()).unwrap(); let agrs_vec: Vec = gossip.try_into().unwrap(); - let subject = DebutHandler::new(Logger::new("test")); + let subject = DebutHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let result = subject.qualifies(&db, agrs_vec.as_slice(), gossip_source); - assert_eq!(result, Qualification::Unmatched); + assert_eq!( + result, + Qualification::Unmatched("Node record already in database".to_string()) + ); } #[test] - fn debut_of_already_connected_node_produces_accepted_result_instead_of_introduction_to_prevent_overconnection( + fn debut_of_already_connected_node_produces_debut_back_instead_of_introduction_to_prevent_overconnection( ) { let src_root = make_node_record(1234, true); let mut src_db = db_from_node(&src_root); @@ -1718,7 +2294,7 @@ mod tests { .build() .try_into() .unwrap(); - let subject = DebutHandler::new(Logger::new("test")); + let subject = DebutHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let result = subject.handle( &dest_cryptde, @@ -1728,7 +2304,57 @@ mod tests { make_default_neighborhood_metadata(), ); - assert_eq!(result, GossipAcceptanceResult::Accepted); + let expected_debut_back_gossip = GossipBuilder::new(&dest_db) + .node(dest_root.public_key(), true) + .build(); + assert_eq!( + result, + vec![GossipAcceptanceResult::Reply( + expected_debut_back_gossip, + src_root.public_key().clone(), + src_root.node_addr_opt().unwrap(), + )] + ); + } + + #[test] + fn debut_is_rejected_when_validation_fails() { + init_test_logging(); + let test_name = "debut_is_rejected_when_validation_fails"; + let root_node = make_node_record(1234, true); + let root_node_cryptde = CryptDENull::from(&root_node.public_key(), TEST_DEFAULT_CHAIN); + let mut src_db = db_from_node(&root_node); + let agrs_vec: Vec = GossipBuilder::new(&src_db) + .node(root_node.public_key(), true) + .build() + .try_into() + .unwrap(); + let subject = DebutHandler::new( + &RatePackLimits::new(RatePack::new(0, 0, 0, 0), RatePack::new(0, 0, 0, 0)), + Logger::new(test_name), + ); + + let result = subject.handle( + &root_node_cryptde, + &mut src_db, + agrs_vec, + root_node.node_addr_opt().clone().unwrap().into(), + make_default_neighborhood_metadata(), + ); + + let message = r#"Debut from AQIDBA at 1.2.3.4:1234 rejected due to rate pack limit violation: ConfiguratorError { param_errors: [ParamError { parameter: "rate-pack", reason: "Value of routing_byte_rate (1235) is above the maximum allowed (0)" }, ParamError { parameter: "rate-pack", reason: "Value of routing_service_rate (1434) is above the maximum allowed (0)" }, ParamError { parameter: "rate-pack", reason: "Value of exit_byte_rate (1237) is above the maximum allowed (0)" }, ParamError { parameter: "rate-pack", reason: "Value of exit_service_rate (1634) is above the maximum allowed (0)" }] }"#.to_string(); + assert_eq!( + result, + vec![GossipAcceptanceResult::Ban(Malefactor::new( + Some(root_node.public_key().clone()), + Some(root_node.node_addr_opt().unwrap().ip_addr()), + Some(root_node.earning_wallet()), + None, + message.clone() + ))] + ); + TestLogHandler::new() + .exists_log_containing(format!("WARN: {}: {}", test_name, message).as_str()); } #[test] @@ -1744,7 +2370,7 @@ mod tests { half_neighbor_debutant.public_key(), ); let logger = Logger::new("Debut test"); - let subject = DebutHandler::new(logger); + let subject = DebutHandler::new(&RatePackLimits::test_default(), logger); let neighborhood_metadata = make_default_neighborhood_metadata(); let counter_debut = subject @@ -1767,9 +2393,135 @@ mod tests { assert_eq!(dest_node_addr, &new_debutant.node_addr_opt().unwrap()); } + #[test] + fn if_we_have_neighbors_but_none_qualify_for_introduction_or_pass_and_does_not_half_neighbor_us_then_debut_back( + ) { + let test_name = "if_we_have_neighbors_but_none_qualify_for_introduction_or_pass_and_does_not_half_neighbor_us_then_debut_back"; + let dest_root = make_node_record(1234, true); + let root_node_cryptde = CryptDENull::from(&dest_root.public_key(), TEST_DEFAULT_CHAIN); + let one_common_neighbor = make_node_record(4321, true); // can't introduce this one; debutant already knows it + let another_common_neighbor = make_node_record(5432, true); // can't introduce this one; debutant already knows it + let mut dest_db = db_from_node(&dest_root); + dest_db.add_node(one_common_neighbor.clone()).unwrap(); + dest_db + .add_arbitrary_full_neighbor(dest_root.public_key(), one_common_neighbor.public_key()); + dest_db.add_node(another_common_neighbor.clone()).unwrap(); + dest_db.add_arbitrary_full_neighbor( + dest_root.public_key(), + another_common_neighbor.public_key(), + ); + let original_version = dest_db.root().version(); + + let mut debutant = make_node_record(4567, true); + debutant + .inner + .neighbors + .insert(one_common_neighbor.public_key().clone()); + debutant + .inner + .neighbors + .insert(another_common_neighbor.public_key().clone()); + let logger = Logger::new(test_name); + let subject = DebutHandler::new(&RatePackLimits::test_default(), logger); + let neighborhood_metadata = make_default_neighborhood_metadata(); + + let result = subject + .try_accept_debut( + &root_node_cryptde, + &mut dest_db, + &AccessibleGossipRecord::from(&debutant), + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(4, 5, 6, 7)), 4567), + neighborhood_metadata.user_exit_preferences_opt, + ) + .unwrap(); + + assert_eq!( + result, + GossipAcceptanceResult::Reply( + GossipBuilder::new(&dest_db) + .node(dest_db.root().public_key(), true) + .build(), + debutant.public_key().clone(), + debutant + .node_addr_opt() + .expect("Debutant should have NodeAddr"), + ) + ); + assert!(dest_db + .root() + .half_neighbor_keys() + .contains(debutant.public_key())); + assert_eq!(dest_db.root().version(), original_version + 1); + root_node_cryptde.verify_signature( + dest_db.root().signed_gossip(), + dest_db.root().signature(), + dest_db.root().public_key(), + ); + } + + #[test] + fn if_we_have_neighbors_but_none_qualify_for_introduction_or_pass_and_half_neighbors_us_then_standard_gossip( + ) { + let test_name = "if_we_have_neighbors_but_none_qualify_for_introduction_or_pass_and_half_neighbors_us_then_standard_gossip"; + let dest_root = make_node_record(1234, true); + let root_node_cryptde = CryptDENull::from(&dest_root.public_key(), TEST_DEFAULT_CHAIN); + let one_common_neighbor = make_node_record(4321, true); // can't introduce this one; debutant already knows it + let another_common_neighbor = make_node_record(5432, true); // can't introduce this one; debutant already knows it + let mut dest_db = db_from_node(&dest_root); + dest_db.add_node(one_common_neighbor.clone()).unwrap(); + dest_db + .add_arbitrary_full_neighbor(dest_root.public_key(), one_common_neighbor.public_key()); + dest_db.add_node(another_common_neighbor.clone()).unwrap(); + dest_db.add_arbitrary_full_neighbor( + dest_root.public_key(), + another_common_neighbor.public_key(), + ); + let original_version = dest_db.root().version(); + + let mut debutant = make_node_record(4567, true); + debutant + .inner + .neighbors + .insert(one_common_neighbor.public_key().clone()); + debutant + .inner + .neighbors + .insert(another_common_neighbor.public_key().clone()); + debutant + .inner + .neighbors + .insert(dest_root.public_key().clone()); + let logger = Logger::new(test_name); + let subject = DebutHandler::new(&RatePackLimits::test_default(), logger); + let neighborhood_metadata = make_default_neighborhood_metadata(); + + let result = subject + .try_accept_debut( + &root_node_cryptde, + &mut dest_db, + &AccessibleGossipRecord::from(&debutant), + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(4, 5, 6, 7)), 4567), + neighborhood_metadata.user_exit_preferences_opt, + ) + .unwrap(); + + assert_eq!(result, GossipAcceptanceResult::Accepted); + assert!(dest_db + .root() + .half_neighbor_keys() + .contains(debutant.public_key())); + assert_eq!(dest_db.root().version(), original_version + 1); + root_node_cryptde.verify_signature( + dest_db.root().signed_gossip(), + dest_db.root().signature(), + dest_db.root().public_key(), + ); + } + #[test] fn proper_pass_is_identified_and_processed() { - let (gossip, pass_target, gossip_source) = make_pass(2345); + let (gossip, pass_target, gossip_source) = + make_pass(2345, NeighborhoodModeLight::Standard.into()); let subject = PassHandler::new(); let mut dest_db = make_meaningless_db(); let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); @@ -1789,18 +2541,53 @@ mod tests { .node(dest_db.root().public_key(), true) .build(); assert_eq!( - GossipAcceptanceResult::Reply( + handle_result, + vec![GossipAcceptanceResult::Reply( debut, pass_target.public_key().clone(), pass_target.node_addr_opt().unwrap().clone(), - ), - handle_result + )], + ); + } + + #[test] + fn pass_with_node_that_does_not_accept_connections_is_rejected() { + let root_node = make_node_record(1234, true); // irrelevant + let mut db = db_from_node(&root_node); // irrelevant + let (gossip, _pass_target, gossip_source) = + make_pass(2345, NeighborhoodModeLight::OriginateOnly.into()); + let agrs_vec: Vec = gossip.try_into().unwrap(); + let subject = PassHandler::new(); + + let result = subject.qualifies(&mut db, agrs_vec.as_slice(), gossip_source); + + assert_eq!( + result, + Qualification::Unmatched("Pass Node record always has a NodeAddr".to_string()) + ); + } + + #[test] + fn pass_with_node_that_does_not_route_data_is_rejected() { + let root_node = make_node_record(1234, true); // irrelevant + let mut db = db_from_node(&root_node); // irrelevant + let (gossip, _pass_target, gossip_source) = + make_pass(2345, NeighborhoodModeLight::ConsumeOnly.into()); + let agrs_vec: Vec = gossip.try_into().unwrap(); + let subject = PassHandler::new(); + + let result = subject.qualifies(&mut db, agrs_vec.as_slice(), gossip_source); + + assert_eq!( + result, + Qualification::Unmatched("Pass Node record always has a NodeAddr".to_string()) ); } #[test] fn pass_without_node_addr_is_rejected() { - let (mut gossip, _, gossip_source) = make_pass(2345); + let (mut gossip, _, gossip_source) = + make_pass(2345, NeighborhoodModeLight::Standard.into()); gossip.node_records[0].node_addr_opt = None; let subject = PassHandler::new(); let agrs_vec: Vec = gossip.try_into().unwrap(); @@ -1808,16 +2595,15 @@ mod tests { let result = subject.qualifies(&make_meaningless_db(), agrs_vec.as_slice(), gossip_source); assert_eq!( - Qualification::Malformed( - "Pass from 200.200.200.200:2000 to AgMEBQ did not contain NodeAddr".to_string() - ), - result + result, + Qualification::Unmatched("Pass Node record always has a NodeAddr".to_string()), ); } #[test] fn pass_without_node_addr_ports_is_rejected() { - let (mut gossip, _, gossip_source) = make_pass(2345); + let (mut gossip, _, gossip_source) = + make_pass(2345, NeighborhoodModeLight::Standard.into()); gossip.node_records[0].node_addr_opt = Some(NodeAddr::new(&IpAddr::from_str("1.2.3.4").unwrap(), &[])); let subject = PassHandler::new(); @@ -1826,38 +2612,103 @@ mod tests { let result = subject.qualifies(&make_meaningless_db(), agrs_vec.as_slice(), gossip_source); assert_eq!( - Qualification::Malformed( - "Pass from 200.200.200.200:2000 to AgMEBQ at 1.2.3.4 contained NodeAddr with no ports" - .to_string() - ), - result + result, + Qualification::Malformed(Malefactor::new( + Some(agrs_vec[0].inner.public_key.clone()), + Some(gossip_source.ip()), + Some(agrs_vec[0].inner.earning_wallet.clone()), + None, + "Pass contained NodeAddr with no ports".to_string() + )), ); } #[test] fn gossip_containing_other_than_two_records_is_not_an_introduction() { - let (gossip, _, gossip_source) = make_debut(2345, Mode::Standard); - let subject = IntroductionHandler::new(Logger::new("test")); + let (gossip, _, gossip_source) = make_debut(2345, NeighborhoodModeLight::Standard.into()); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs_vec: Vec = gossip.try_into().unwrap(); let result = subject.qualifies(&make_meaningless_db(), agrs_vec.as_slice(), gossip_source); - assert_eq!(Qualification::Unmatched, result); + assert_eq!( + Qualification::Unmatched("Introduction has 2 Node records, not 1".to_string()), + result + ); } #[test] - fn introduction_where_introducee_is_in_the_database_is_unmatched() { + fn introduction_is_still_matched_even_if_receiver_has_half_neighborship_with_introducer() { let (gossip, gossip_source) = make_introduction(2345, 3456); - let not_introducee = make_node_record(3456, true); + let introducer_in_target_db = make_node_record(2345, true); + let introducee_in_target_db = make_node_record(3456, true); let dest_root = make_node_record(7878, true); let mut dest_db = db_from_node(&dest_root); - dest_db.add_node(not_introducee.clone()).unwrap(); - let subject = IntroductionHandler::new(Logger::new("test")); + dest_db.add_node(introducer_in_target_db.clone()).unwrap(); + dest_db + .add_half_neighbor(introducer_in_target_db.public_key()) + .unwrap(); //disqualifying + dest_db.add_node(introducee_in_target_db.clone()).unwrap(); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs_vec: Vec = gossip.try_into().unwrap(); let result = subject.qualifies(&dest_db, agrs_vec.as_slice(), gossip_source); - assert_eq!(Qualification::Unmatched, result); + assert_eq!(result, Qualification::Matched); + } + + #[test] + fn introduction_is_matched_if_receiver_has_half_neighborship_with_introducee_but_not_introducer( + ) { + let (gossip, gossip_source) = make_introduction(2345, 3456); + let introducer_in_target_db = make_node_record(2345, true); + let introducee_in_target_db = make_node_record(3456, true); + let dest_root = make_node_record(7878, true); + let mut dest_db = db_from_node(&dest_root); + dest_db.add_node(introducer_in_target_db.clone()).unwrap(); + dest_db.add_node(introducee_in_target_db.clone()).unwrap(); + dest_db + .add_half_neighbor(introducee_in_target_db.public_key()) + .unwrap(); //disqualifying + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); + let agrs_vec: Vec = gossip.try_into().unwrap(); + + let result = subject.qualifies(&dest_db, agrs_vec.as_slice(), gossip_source); + + assert_eq!(result, Qualification::Matched); + } + + #[test] + fn introduction_is_unmatched_if_receiver_has_half_neighborships_with_both_introducer_and_introducee( + ) { + let (gossip, gossip_source) = make_introduction(2345, 3456); + let introducer_in_target_db = make_node_record(2345, true); + let introducee_in_target_db = make_node_record(3456, true); + let dest_root = make_node_record(7878, true); + let mut dest_db = db_from_node(&dest_root); + dest_db.add_node(introducer_in_target_db.clone()).unwrap(); + dest_db + .add_half_neighbor(introducer_in_target_db.public_key()) + .unwrap(); //disqualifying + dest_db.add_node(introducee_in_target_db.clone()).unwrap(); + dest_db + .add_half_neighbor(introducee_in_target_db.public_key()) + .unwrap(); //disqualifying + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); + let agrs_vec: Vec = gossip.try_into().unwrap(); + + let result = subject.qualifies(&dest_db, agrs_vec.as_slice(), gossip_source); + + assert_eq!( + result, + Qualification::Unmatched( + "Introducer and introducee are already our half-neighbors".to_string() + ) + ); } #[test] @@ -1866,12 +2717,18 @@ mod tests { let dest_root = make_node_record(7878, true); let dest_db = db_from_node(&dest_root); gossip.node_records[0].node_addr_opt = None; - let subject = IntroductionHandler::new(Logger::new("test")); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs_vec: Vec = gossip.try_into().unwrap(); let result = subject.qualifies(&dest_db, agrs_vec.as_slice(), gossip_source); - assert_eq!(Qualification::Unmatched, result); + assert_eq!( + Qualification::Unmatched( + "Both Node records in Introduction must have NodeAddr".to_string() + ), + result + ); } #[test] @@ -1881,14 +2738,21 @@ mod tests { let dest_db = db_from_node(&dest_root); gossip.node_records[0].node_addr_opt = Some(NodeAddr::new(&IpAddr::from_str("2.3.4.5").unwrap(), &[])); - let subject = IntroductionHandler::new(Logger::new("test")); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs_vec: Vec = gossip.try_into().unwrap(); let result = subject.qualifies(&dest_db, agrs_vec.as_slice(), gossip_source); assert_eq!( - Qualification::Malformed("Introducer AgMEBQ from 2.3.4.5 has no ports".to_string()), - result + result, + Qualification::Malformed(Malefactor::new( + Some(agrs_vec[0].inner.public_key.clone()), + Some(gossip_source.ip()), + Some(agrs_vec[0].inner.earning_wallet.clone()), + None, + "Introducer has no ports".to_string() + )), ); } @@ -1898,12 +2762,18 @@ mod tests { let dest_root = make_node_record(7878, true); let dest_db = db_from_node(&dest_root); gossip.node_records[1].node_addr_opt = None; - let subject = IntroductionHandler::new(Logger::new("test")); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs_vec: Vec = gossip.try_into().unwrap(); let result = subject.qualifies(&dest_db, agrs_vec.as_slice(), gossip_source); - assert_eq!(Qualification::Unmatched, result); + assert_eq!( + Qualification::Unmatched( + "Both Node records in Introduction must have NodeAddr".to_string() + ), + result + ); } #[test] @@ -1913,17 +2783,21 @@ mod tests { let dest_db = db_from_node(&dest_root); gossip.node_records[1].node_addr_opt = Some(NodeAddr::new(&IpAddr::from_str("3.4.5.6").unwrap(), &[])); - let subject = IntroductionHandler::new(Logger::new("test")); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs_vec: Vec = gossip.try_into().unwrap(); let result = subject.qualifies(&dest_db, agrs_vec.as_slice(), gossip_source); assert_eq!( - Qualification::Malformed( - "Introducer AgMEBQ from 2.3.4.5 introduced AwQFBg from 3.4.5.6 with no ports" - .to_string() - ), - result + result, + Qualification::Malformed(Malefactor::new( + Some(agrs_vec[0].inner.public_key.clone()), + Some(gossip_source.ip()), + Some(agrs_vec[0].inner.earning_wallet.clone()), + None, + "Introducer introduced introducee with no ports".to_string() + )), ); } @@ -1936,12 +2810,22 @@ mod tests { &IpAddr::from_str("4.5.6.7").unwrap(), &[4567], )); - let subject = IntroductionHandler::new(Logger::new("test")); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs_vec: Vec = gossip.try_into().unwrap(); let result = subject.qualifies(&dest_db, agrs_vec.as_slice(), gossip_source); - assert_eq!(Qualification::Malformed("In Introduction, neither AgMEBQ from 4.5.6.7 nor AwQFBg from 3.4.5.6 claims the source IP 2.3.4.5".to_string()), result); + assert_eq!( + result, + Qualification::Malformed(Malefactor::new( + None, + Some(gossip_source.ip()), + None, + None, + "In Introduction, neither Node record claims the source IP".to_string() + )), + ); } #[test] @@ -1953,16 +2837,21 @@ mod tests { &IpAddr::from_str("2.3.4.5").unwrap(), &[2345], )); - let subject = IntroductionHandler::new(Logger::new("test")); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs_vec: Vec = gossip.try_into().unwrap(); let result = subject.qualifies(&dest_db, agrs_vec.as_slice(), gossip_source); assert_eq!( - Qualification::Malformed( - "Introducer AgMEBQ and introducee AwQFBg both claim 2.3.4.5".to_string() - ), - result + result, + Qualification::Malformed(Malefactor::new( + Some(agrs_vec[0].inner.public_key.clone()), + Some(gossip_source.ip()), + Some(agrs_vec[0].inner.earning_wallet.clone()), + None, + "Introducer claims the same IP address as its introducee".to_string() + )), ); } @@ -1971,7 +2860,8 @@ mod tests { let (gossip, gossip_source) = make_introduction(2345, 3456); let dest_root = make_node_record(7878, true); let dest_db = db_from_node(&dest_root); - let subject = IntroductionHandler::new(Logger::new("test")); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let mut agrs: Vec = gossip.try_into().unwrap(); agrs[0].inner.public_key = dest_root.public_key().clone(); let introducer_key = &agrs[0].inner.public_key; @@ -1979,11 +2869,14 @@ mod tests { let result = subject.qualifies(&dest_db, &agrs, gossip_source); assert_eq!( - Qualification::Malformed(format!( - "Introducer {} claims local Node's public key", - introducer_key + result, + Qualification::Malformed(Malefactor::new( + Some(introducer_key.clone()), + Some(gossip_source.ip()), + Some(agrs[0].inner.earning_wallet.clone()), + None, + "Introducer claims local Node's public key".to_string() )), - result ); } @@ -1991,67 +2884,23 @@ mod tests { fn introduction_where_introducer_has_local_ip_address_is_malformed() { let (gossip, _) = make_introduction(2345, 3456); let dest_root = make_node_record(7878, true); - let dest_db = db_from_node(&dest_root); - let subject = IntroductionHandler::new(Logger::new("test")); - let mut agrs: Vec = gossip.try_into().unwrap(); - agrs[0].node_addr_opt = dest_root.node_addr_opt(); - let introducer_key = &agrs[0].inner.public_key; - - let result = subject.qualifies(&dest_db, &agrs, dest_root.node_addr_opt().unwrap().into()); - - assert_eq!( - Qualification::Malformed(format!( - "Introducer {} claims to be at local Node's IP address", - introducer_key - )), - result - ); - } - - #[test] - fn introduction_that_tries_to_change_immutable_characteristics_of_introducer_is_suspicious() { - let (gossip, gossip_source) = make_introduction(2345, 3456); - let dest_root = make_node_record(7878, true); - let mut dest_db = db_from_node(&dest_root); - let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); - let subject = IntroductionHandler::new(Logger::new("test")); - let agrs: Vec = gossip.try_into().unwrap(); - let introducer_key = &agrs[0].inner.public_key; - dest_db.add_node(NodeRecord::from(&agrs[0])).unwrap(); - dest_db - .node_by_key_mut(introducer_key) - .unwrap() - .set_version(0); - dest_db - .node_by_key_mut(introducer_key) - .unwrap() - .force_node_addr(&NodeAddr::from( - &SocketAddr::from_str("4.5.6.7:4567").unwrap(), - )); - dest_db.resign_node(introducer_key); - let introducer_before_gossip = dest_db.node_by_key(introducer_key).unwrap().clone(); - let before = time_t_timestamp(); - - let qualifies_result = subject.qualifies(&dest_db, &agrs, gossip_source); - let handle_result = subject.handle( - &cryptde, - &mut dest_db, - agrs.clone(), - gossip_source, - make_default_neighborhood_metadata(), - ); + let dest_db = db_from_node(&dest_root); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); + let mut agrs: Vec = gossip.try_into().unwrap(); + agrs[0].node_addr_opt = dest_root.node_addr_opt(); + + let result = subject.qualifies(&dest_db, &agrs, dest_root.node_addr_opt().unwrap().into()); - let after = time_t_timestamp(); - assert_eq!(qualifies_result, Qualification::Matched); assert_eq!( - handle_result, - GossipAcceptanceResult::Ban(format!("Introducer {} tried changing immutable characteristic: Updating a NodeRecord must not change its node_addr_opt: 4.5.6.7:4567 -> 2.3.4.5:2345", introducer_key)), - ); - assert_node_records_eq( - dest_db.node_by_key_mut(introducer_key).unwrap(), - &introducer_before_gossip, - before, - after, + result, + Qualification::Malformed(Malefactor::new( + Some(agrs[0].inner.public_key.clone()), + Some(dest_root.node_addr_opt().unwrap().ip_addr()), + Some(agrs[0].inner.earning_wallet.clone()), + None, + "Introducer claims to be at local Node's IP address".to_string() + )), ); } @@ -2060,7 +2909,8 @@ mod tests { let (gossip, gossip_source) = make_introduction(2345, 3456); let dest_root = make_node_record_f(7878, false, false, true); let dest_db = db_from_node(&dest_root); - let subject = IntroductionHandler::new(Logger::new("test")); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs: Vec = gossip.try_into().unwrap(); let result = subject.qualifies(&dest_db, &agrs, gossip_source); @@ -2074,7 +2924,8 @@ mod tests { let dest_root = make_node_record(7878, true); let mut dest_db = db_from_node(&dest_root); let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); - let subject = IntroductionHandler::new(Logger::new("test")); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs: Vec = gossip.try_into().unwrap(); let mut expected_introducer = NodeRecord::from(&agrs[0]); expected_introducer.metadata.country_undesirability = COUNTRY_UNDESIRABILITY_FACTOR; @@ -2104,12 +2955,15 @@ mod tests { .node(dest_db.root().public_key(), true) .build(); assert_eq!( - GossipAcceptanceResult::Reply( - debut, - agrs[1].inner.public_key.clone(), - agrs[1].node_addr_opt.clone().unwrap(), - ), - handle_result + handle_result, + vec![ + GossipAcceptanceResult::Accepted, + GossipAcceptanceResult::Reply( + debut, + agrs[1].inner.public_key.clone(), + agrs[1].node_addr_opt.clone().unwrap(), + ) + ], ); let result_introducer: &NodeRecord = dest_db.node_by_key(&agrs[0].inner.public_key).unwrap(); @@ -2134,7 +2988,8 @@ mod tests { dest_db.add_arbitrary_full_neighbor(dest_root.public_key(), &key); } let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); - let subject = IntroductionHandler::new(Logger::new("test")); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs: Vec = gossip.try_into().unwrap(); let handle_result = subject.handle( @@ -2145,7 +3000,7 @@ mod tests { make_default_neighborhood_metadata(), ); - assert_eq!(handle_result, GossipAcceptanceResult::Ignored); + assert_eq!(handle_result, vec![]); } #[test] @@ -2155,7 +3010,8 @@ mod tests { let dest_root = make_node_record(7878, true); let mut dest_db = db_from_node(&dest_root); let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); - let subject = IntroductionHandler::new(Logger::new("test")); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs: Vec = gossip.try_into().unwrap(); dest_db.add_node(NodeRecord::from(&agrs[0])).unwrap(); dest_db @@ -2178,12 +3034,15 @@ mod tests { .node(dest_db.root().public_key(), true) .build(); assert_eq!( - GossipAcceptanceResult::Reply( - debut, - agrs[1].inner.public_key.clone(), - agrs[1].node_addr_opt.clone().unwrap(), - ), - handle_result + handle_result, + vec![ + GossipAcceptanceResult::Accepted, + GossipAcceptanceResult::Reply( + debut, + agrs[1].inner.public_key.clone(), + agrs[1].node_addr_opt.clone().unwrap(), + ) + ], ); let result_introducer: &NodeRecord = dest_db.node_by_key(&agrs[0].inner.public_key).unwrap(); @@ -2192,13 +3051,13 @@ mod tests { expected_introducer.resign(); assert_eq!(result_introducer, &expected_introducer); assert_eq!( - true, dest_db .root() - .has_half_neighbor(expected_introducer.public_key()) + .has_half_neighbor(expected_introducer.public_key()), + true, ); - assert_eq!(1, dest_db.root().version()); - assert_eq!(None, dest_db.node_by_key(&agrs[1].inner.public_key)); + assert_eq!(dest_db.root().version(), 1); + assert_eq!(dest_db.node_by_key(&agrs[1].inner.public_key), None); } #[test] @@ -2207,7 +3066,6 @@ mod tests { let (gossip, gossip_source) = make_introduction(2345, 3456); let dest_root = make_node_record(7878, true); let mut dest_db = db_from_node(&dest_root); - // These don't count because they're half-only neighbors. Will they be ignored? for idx in 0..MAX_DEGREE { let half_neighbor_key = &dest_db .add_node(make_node_record(4000 + idx as u16, true)) @@ -2215,7 +3073,8 @@ mod tests { dest_db.add_arbitrary_half_neighbor(dest_root.public_key(), half_neighbor_key); } let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); - let subject = IntroductionHandler::new(Logger::new("test")); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let agrs: Vec = gossip.try_into().unwrap(); dest_db.add_node(NodeRecord::from(&agrs[0])).unwrap(); dest_db.add_arbitrary_half_neighbor(dest_root.public_key(), &agrs[0].inner.public_key); @@ -2234,12 +3093,12 @@ mod tests { .node(dest_db.root().public_key(), true) .build(); assert_eq!( - GossipAcceptanceResult::Reply( + handle_result, + vec![GossipAcceptanceResult::Reply( debut, agrs[1].inner.public_key.clone(), agrs[1].node_addr_opt.clone().unwrap(), - ), - handle_result + )], ); let result_introducer: &NodeRecord = @@ -2249,13 +3108,119 @@ mod tests { expected_introducer.resign(); assert_eq!(result_introducer, &expected_introducer); assert_eq!( - true, dest_db .root() - .has_half_neighbor(expected_introducer.public_key()) + .has_half_neighbor(expected_introducer.public_key()), + true, + ); + assert_eq!(dest_db.root().version(), 0); + assert_eq!(dest_db.node_by_key(&agrs[1].inner.public_key), None); + } + + #[test] + fn introducer_that_fails_validation_is_rejected() { + init_test_logging(); + let test_name = "introducer_that_fails_validation_is_rejected"; + let (gossip, gossip_source) = make_introduction(2345, 3456); + let dest_root = make_node_record(7878, true); + let mut dest_db = db_from_node(&dest_root); + for idx in 0..MAX_DEGREE { + let half_neighbor_key = &dest_db + .add_node(make_node_record(4000 + idx as u16, true)) + .unwrap(); + dest_db.add_arbitrary_half_neighbor(dest_root.public_key(), half_neighbor_key); + } + let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); + let subject = IntroductionHandler::new( + &RatePackLimits::new( + RatePack::new(100, 100, 100, 100), + RatePack::new(u64::MAX, u64::MAX, u64::MAX, u64::MAX), + ), + Logger::new(test_name), + ); + let mut agrs: Vec = gossip.try_into().unwrap(); + agrs[0].inner.rate_pack = RatePack::new(0, 0, 0, 0); // Invalid rate pack + dest_db.add_node(NodeRecord::from(&agrs[0])).unwrap(); + dest_db.add_arbitrary_half_neighbor(dest_root.public_key(), &agrs[0].inner.public_key); + + let handle_result = subject.handle( + &cryptde, + &mut dest_db, + agrs.clone(), + gossip_source, + make_default_neighborhood_metadata(), + ); + + let message = format!("Introducer {} from {} rejected due to rate pack limit violation: ConfiguratorError {{ param_errors: [ParamError {{ parameter: \"rate-pack\", reason: \"Value of routing_byte_rate (0) is below the minimum allowed (100)\" }}, ParamError {{ parameter: \"rate-pack\", reason: \"Value of routing_service_rate (0) is below the minimum allowed (100)\" }}, ParamError {{ parameter: \"rate-pack\", reason: \"Value of exit_byte_rate (0) is below the minimum allowed (100)\" }}, ParamError {{ parameter: \"rate-pack\", reason: \"Value of exit_service_rate (0) is below the minimum allowed (100)\" }}] }}", agrs[0].inner.public_key, gossip_source); + // If we decide the introducer is a malefactor, we won't accept the introducee from him. + // However, the introducee may be perfectly innocent; so we don't want to ban the + // introducee in case he's introduced later by somebody we do trust. + assert_eq!( + handle_result, + vec![GossipAcceptanceResult::Ban(Malefactor::new( + Some(agrs[0].inner.public_key.clone()), + Some(agrs[0].node_addr_opt.as_ref().unwrap().ip_addr()), + Some(agrs[0].inner.earning_wallet.clone()), + None, + message.clone() + ))] + ); + TestLogHandler::new() + .exists_log_containing(format!("WARN: {}: {}", test_name, message).as_str()); + } + + #[test] + fn introducee_that_fails_validation_is_rejected() { + init_test_logging(); + let test_name = "introducee_that_fails_validation_is_rejected"; + let (gossip, gossip_source) = make_introduction(2345, 3456); + let dest_root = make_node_record(7878, true); + let mut dest_db = db_from_node(&dest_root); + // These don't count because they're half-only neighbors. Will they be ignored? + for idx in 0..MAX_DEGREE { + let half_neighbor_key = &dest_db + .add_node(make_node_record(4000 + idx as u16, true)) + .unwrap(); + dest_db.add_arbitrary_half_neighbor(dest_root.public_key(), half_neighbor_key); + } + let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); + let subject = IntroductionHandler::new( + &RatePackLimits::new( + RatePack::new(100, 100, 100, 100), + RatePack::new(u64::MAX, u64::MAX, u64::MAX, u64::MAX), + ), + Logger::new(test_name), + ); + let mut agrs: Vec = gossip.try_into().unwrap(); + agrs[1].inner.rate_pack = RatePack::new(0, 0, 0, 0); // Invalid rate pack + dest_db.add_node(NodeRecord::from(&agrs[0])).unwrap(); + dest_db.add_arbitrary_half_neighbor(dest_root.public_key(), &agrs[0].inner.public_key); + + let handle_result = subject.handle( + &cryptde, + &mut dest_db, + agrs.clone(), + gossip_source, + make_default_neighborhood_metadata(), ); - assert_eq!(0, dest_db.root().version()); - assert_eq!(None, dest_db.node_by_key(&agrs[1].inner.public_key)); + + let message = format!( + "Introducee {} at {} rejected due to rate pack limit violation: ConfiguratorError {{ param_errors: [ParamError {{ parameter: \"rate-pack\", reason: \"Value of routing_byte_rate (0) is below the minimum allowed (100)\" }}, ParamError {{ parameter: \"rate-pack\", reason: \"Value of routing_service_rate (0) is below the minimum allowed (100)\" }}, ParamError {{ parameter: \"rate-pack\", reason: \"Value of exit_byte_rate (0) is below the minimum allowed (100)\" }}, ParamError {{ parameter: \"rate-pack\", reason: \"Value of exit_service_rate (0) is below the minimum allowed (100)\" }}] }}", + agrs[1].inner.public_key, + agrs[1].node_addr_opt.as_ref().unwrap().ip_addr() + ); + assert_eq!( + handle_result, + vec![GossipAcceptanceResult::Ban(Malefactor::new( + Some(agrs[1].inner.public_key.clone()), + Some(agrs[1].node_addr_opt.as_ref().unwrap().ip_addr()), + Some(agrs[1].inner.earning_wallet.clone()), + None, + message.clone() + ))], + ); + TestLogHandler::new() + .exists_log_containing(format!("WARN: {}: {}", test_name, message).as_str()); } #[test] @@ -2316,7 +3281,8 @@ mod tests { .node(node_a.public_key(), false) .node(node_b.public_key(), false) .build(); - let subject = StandardGossipHandler::new(Logger::new("test")); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let gossip_source: SocketAddr = src_node.node_addr_opt().unwrap().into(); let gossip_vec: Vec = gossip.try_into().unwrap(); @@ -2342,7 +3308,8 @@ mod tests { .node(node_a.public_key(), false) .node(dest_node.public_key(), false) .build(); - let subject = StandardGossipHandler::new(Logger::new("test")); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let gossip_source: SocketAddr = src_node.node_addr_opt().unwrap().into(); let gossip_vec: Vec = gossip.try_into().unwrap(); @@ -2350,10 +3317,13 @@ mod tests { assert_eq!( result, - Qualification::Malformed( - "Standard Gossip from 1.2.3.4:1234 contains a record with this Node's public key" - .to_string() - ), + Qualification::Malformed(Malefactor::new( + Some(dest_node.public_key().clone()), + None, + Some(dest_node.earning_wallet().clone()), + None, + "Standard Gossip contains a record claiming this Node's public key".to_string() + )), ); } @@ -2377,7 +3347,8 @@ mod tests { .node(node_a.public_key(), false) .node(node_b.public_key(), true) .build(); - let subject = StandardGossipHandler::new(Logger::new("test")); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let gossip_source: SocketAddr = src_node.node_addr_opt().unwrap().into(); let gossip_vec: Vec = gossip.try_into().unwrap(); @@ -2385,9 +3356,15 @@ mod tests { assert_eq!( result, - Qualification::Malformed(format!( - "Standard Gossip from 1.2.3.4:1234 contains a record claiming that {} has this Node's IP address", - node_b.public_key() + Qualification::Malformed(Malefactor::new( + None, + Some(gossip_source.ip()), + None, + None, + format!( + "Standard Gossip contains a record claiming that {} resides at this Node's IP address", + node_b.public_key() + ) )), ); } @@ -2415,7 +3392,8 @@ mod tests { &node_a.node_addr_opt().unwrap().ip_addr(), &[4567], )); - let subject = StandardGossipHandler::new(Logger::new("test")); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let gossip_source: SocketAddr = src_node.node_addr_opt().unwrap().into(); let gossip_vec: Vec = gossip.try_into().unwrap(); @@ -2423,9 +3401,13 @@ mod tests { assert_eq!( result, - Qualification::Malformed( - "Standard Gossip from 1.2.3.4:1234 contains multiple records claiming to be from 3.4.5.6".to_string() - ), + Qualification::Malformed(Malefactor::new( + None, + Some(gossip_source.ip()), + None, + None, + "Standard Gossip contains multiple records claiming to be from 3.4.5.6".to_string() + )), ); } @@ -2474,7 +3456,8 @@ mod tests { .node(node_a.public_key(), false) .node(node_b.public_key(), false) .build(); - let subject = StandardGossipHandler::new(Logger::new("test")); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); let agrs_vec: Vec = gossip.try_into().unwrap(); let gossip_source: SocketAddr = src_root.node_addr_opt().unwrap().into(); @@ -2507,7 +3490,7 @@ mod tests { .unwrap() .metadata .country_undesirability, - 0u32 + 0u32, ); assert_eq!( dest_db @@ -2515,13 +3498,13 @@ mod tests { .unwrap() .metadata .country_undesirability, - UNREACHABLE_COUNTRY_PENALTY + UNREACHABLE_COUNTRY_PENALTY, ); - assert_eq!(Qualification::Matched, qualifies_result); - assert_eq!(GossipAcceptanceResult::Accepted, handle_result); + assert_eq!(qualifies_result, Qualification::Matched); + assert_eq!(handle_result, vec![GossipAcceptanceResult::Accepted]); assert_eq!( &src_db.root().inner, - &dest_db.node_by_key(src_root.public_key()).unwrap().inner + &dest_db.node_by_key(src_root.public_key()).unwrap().inner, ); assert!(dest_db.has_full_neighbor(dest_db.root().public_key(), src_db.root().public_key())); assert_eq!( @@ -2546,7 +3529,8 @@ mod tests { This test proves that E is excluded, because the distance of A and E is more than 3 hops. */ - let subject = StandardGossipHandler::new(Logger::new("test")); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let node_a = make_node_record(1111, true); let node_b = make_node_record(2222, true); let node_c = make_node_record(3333, false); @@ -2595,7 +3579,8 @@ mod tests { 2) To find neighbors of neighbors, we'll look into the AGRs. (For Example, B---Y, B---C, and C---D). */ - let subject = StandardGossipHandler::new(Logger::new("test")); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let node_a = make_node_record(1111, true); let node_b = make_node_record(2222, true); let node_c = make_node_record(3333, false); @@ -2648,7 +3633,8 @@ mod tests { init_test_logging(); let test_name = "standard_gossip_handler_can_handle_node_for_which_agr_is_not_found_while_computing_patch"; - let subject = StandardGossipHandler::new(Logger::new(test_name)); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new(test_name)); let node_a = make_node_record(1111, true); let node_b = make_node_record(2222, true); let node_c = make_node_record(3333, false); @@ -2682,7 +3668,7 @@ mod tests { .collect::>(); assert_eq!(patch, expected_hashset); TestLogHandler::new().exists_log_matching(&format!( - "TRACE: {}: While computing patch no AGR record found for public key {:?}", + "TRACE: {}: While computing patch no Node record found for public key {:?}", test_name, node_x.public_key() )); @@ -2703,8 +3689,9 @@ mod tests { */ - let cryptde = CRYPTDE_PAIR.main.as_ref(); - let subject = StandardGossipHandler::new(Logger::new("test")); + let cryptde = GA_CRYPTDE_PAIR.main.as_ref(); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let node_a = make_node_record(1111, true); let node_b = make_node_record(2222, true); let node_c = make_node_record(3333, false); @@ -2749,11 +3736,14 @@ mod tests { make_default_neighborhood_metadata(), ); - assert_eq!(result, GossipAcceptanceResult::Ignored); + assert_eq!(result, vec![]); } fn assert_compute_patch(db_patch_size: u8) { - let subject = StandardGossipHandler::new(Logger::new("assert_compute_patch")); + let subject = StandardGossipHandler::new( + &RatePackLimits::test_default(), + Logger::new("assert_compute_patch"), + ); // one node to finish hops and another node that's outside the patch let nodes_count = db_patch_size as usize + 2; let nodes = make_node_records(nodes_count as u16); @@ -2784,9 +3774,9 @@ mod tests { // This is Standard Gossip, even though it looks like a Debut, // because it's specifically handled by a StandardGossipHandler // instead of the GossipAcceptor (which would identify it as a Debut), - // so the test is unrealistic. Also that the Gossip is ignored because + // so the test is unrealistic. Also, that the Gossip is ignored because // Node B isn't in Node A's patch, which matters to a StandardGossipHandler. - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = GA_CRYPTDE_PAIR.main.as_ref(); let root_node = make_node_record(1111, true); let mut root_db = db_from_node(&root_node); let src_node = make_node_record(2222, true); @@ -2799,7 +3789,8 @@ mod tests { let (cpm_recipient, recording_arc) = make_cpm_recipient(); let mut neighborhood_metadata = make_default_neighborhood_metadata(); neighborhood_metadata.cpm_recipient = cpm_recipient; - let subject = StandardGossipHandler::new(Logger::new("test")); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let system = System::new("test"); let result = subject.handle( @@ -2814,13 +3805,13 @@ mod tests { assert_eq!(system.run(), 0); let recording = recording_arc.lock().unwrap(); assert_eq!(recording.len(), 0); - assert_eq!(result, GossipAcceptanceResult::Ignored); + assert_eq!(result, vec![]); } #[test] fn cpm_is_sent_in_case_full_neighborship_doesn_t_exist_and_is_created() { // Received Reply for Acceptance of Debut Gossip - (false, true) - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = GA_CRYPTDE_PAIR.main.as_ref(); let root_node = make_node_record(1111, true); let mut root_db = db_from_node(&root_node); let src_node = make_node_record(2222, true); @@ -2839,7 +3830,8 @@ mod tests { let (cpm_recipient, recording_arc) = make_cpm_recipient(); let mut neighborhood_metadata = make_default_neighborhood_metadata(); neighborhood_metadata.cpm_recipient = cpm_recipient; - let subject = StandardGossipHandler::new(Logger::new("test")); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let system = System::new("test"); let result = subject.handle( @@ -2852,7 +3844,7 @@ mod tests { System::current().stop(); assert_eq!(system.run(), 0); - assert_eq!(result, GossipAcceptanceResult::Accepted); + assert_eq!(result, vec![GossipAcceptanceResult::Accepted]); let recording = recording_arc.lock().unwrap(); assert_eq!(recording.len(), 1); let received_message = recording.get_record::(0); @@ -2868,7 +3860,7 @@ mod tests { #[test] fn cpm_is_not_sent_in_case_full_neighborship_exists_and_is_destroyed() { // Somebody banned us. (true, false) - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = GA_CRYPTDE_PAIR.main.as_ref(); let root_node = make_node_record(1111, true); let mut root_db = db_from_node(&root_node); let src_node = make_node_record(2222, true); @@ -2886,7 +3878,8 @@ mod tests { let (cpm_recipient, recording_arc) = make_cpm_recipient(); let mut neighborhood_metadata = make_default_neighborhood_metadata(); neighborhood_metadata.cpm_recipient = cpm_recipient; - let subject = StandardGossipHandler::new(Logger::new("test")); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let system = System::new("test"); let result = subject.handle( @@ -2899,7 +3892,7 @@ mod tests { System::current().stop(); assert_eq!(system.run(), 0); - assert_eq!(result, GossipAcceptanceResult::Accepted); + assert_eq!(result, vec![GossipAcceptanceResult::Accepted]); let recording = recording_arc.lock().unwrap(); assert_eq!(recording.len(), 0); } @@ -2907,7 +3900,7 @@ mod tests { #[test] fn cpm_is_not_sent_in_case_full_neighborship_exists_and_continues() { // Standard Gossips received after Neighborship is established (true, true) - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = GA_CRYPTDE_PAIR.main.as_ref(); let root_node = make_node_record(1111, true); let mut root_db = db_from_node(&root_node); let src_node = make_node_record(2222, true); @@ -2927,7 +3920,8 @@ mod tests { let (cpm_recipient, recording_arc) = make_cpm_recipient(); let mut neighborhood_metadata = make_default_neighborhood_metadata(); neighborhood_metadata.cpm_recipient = cpm_recipient; - let subject = StandardGossipHandler::new(Logger::new("test")); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let system = System::new("test"); let result = subject.handle( @@ -2938,61 +3932,349 @@ mod tests { neighborhood_metadata, ); - System::current().stop(); - assert_eq!(system.run(), 0); - assert_eq!(result, GossipAcceptanceResult::Accepted); - let recording = recording_arc.lock().unwrap(); - assert_eq!(recording.len(), 0); + System::current().stop(); + assert_eq!(system.run(), 0); + assert_eq!(result, vec![ + GossipAcceptanceResult::Accepted, + GossipAcceptanceResult::Ban(Malefactor::new( + Some(root_node.public_key().clone()), + Some(root_node.node_addr_opt().unwrap().ip_addr()), + Some(root_node.earning_wallet()), + None, + format!("Node {} at {} sent Standard gossip that contained a record claiming our own public key", + root_node.public_key(), + root_node.node_addr_opt().unwrap().ip_addr() + ) + )) + ]); + let recording = recording_arc.lock().unwrap(); + assert_eq!(recording.len(), 0); + } + + #[test] + fn standard_gossip_handler_doesnt_add_gossipping_node_if_already_max_degree() { + let src_root = make_node_record(1234, true); + let dest_root = make_node_record(2345, true); + let five_neighbors: Vec = (0..MAX_DEGREE as u16) + .into_iter() + .map(|index| make_node_record(5110 + index, true)) + .collect(); + let add_neighbors = |db: &mut NeighborhoodDatabase, count: usize| { + five_neighbors.iter().take(count).for_each(|node| { + db.add_node(node.clone()).unwrap(); + let root_key = db.root().public_key().clone(); + db.add_arbitrary_full_neighbor(&root_key, node.public_key()); + }); + }; + let mut src_db = db_from_node(&src_root); + add_neighbors(&mut src_db, 2); + src_db.add_node(dest_root.clone()).unwrap(); + src_db.add_arbitrary_full_neighbor(five_neighbors[0].public_key(), dest_root.public_key()); + let mut dest_db = db_from_node(&dest_root); + let dest_cryptde = CryptDENull::from(dest_root.public_key(), TEST_DEFAULT_CHAIN); + add_neighbors(&mut dest_db, MAX_DEGREE); + dest_db.add_node(src_root.clone()).unwrap(); + dest_db.add_arbitrary_full_neighbor(five_neighbors[0].public_key(), src_root.public_key()); + let gossip = GossipProducerReal::new() + .produce(&mut src_db, dest_root.public_key()) + .unwrap(); + let subject = make_subject(&dest_cryptde); + + let result = subject.handle( + &mut dest_db, + gossip.try_into().unwrap(), + src_root.node_addr_opt().clone().unwrap().into(), + make_default_neighborhood_metadata(), + ); + + assert_eq!(result, vec![]); + } + + #[test] + fn standard_gossip_from_node_that_accepts_connections_but_does_not_describe_gossip_source_is_rejected( + ) { + /* + Destination Node ==> + S---D + + Source Node ==> + S---D + + The source node(S) will send Gossip containing no information about + itself, which will get it banned by IP. + */ + init_test_logging(); + let test_name = "standard_gossip_that_does_not_describe_gossip_source_is_rejected"; + let src_root = make_node_record(1234, true); + let dest_root = make_node_record(2345, true); + let mut src_db = db_from_node(&src_root); + let mut dest_db = db_from_node(&dest_root); + dest_db.add_node(src_root.clone()).unwrap(); + dest_db.add_arbitrary_full_neighbor(dest_root.public_key(), src_root.public_key()); + src_db.add_node(dest_db.root().clone()).unwrap(); + src_db.add_arbitrary_full_neighbor(src_root.public_key(), dest_root.public_key()); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new(test_name)); + let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); + let gossip_source: SocketAddr = src_root.node_addr_opt().unwrap().into(); + let (cpm_recipient, _) = make_cpm_recipient(); + let mut neighborhood_metadata = make_default_neighborhood_metadata(); + neighborhood_metadata.cpm_recipient = cpm_recipient; + + let handle_result = subject.handle( + &cryptde, + &mut dest_db, + vec![], + gossip_source, + neighborhood_metadata, + ); + + let message = format!( + "Node at {} sent Standard gossip without a record describing itself", + gossip_source.ip(), + ); + assert_eq!( + handle_result, + vec![GossipAcceptanceResult::Ban(Malefactor::new( + None, + Some(gossip_source.ip()), + None, + None, + message.clone() + )),] + ); + TestLogHandler::new().exists_log_containing(&format!("WARN: {}: {}", test_name, message)); + } + + #[test] + fn standard_gossip_from_node_that_does_not_accept_connections_is_accepted() { + /* + Destination Node ==> + S---D + + Source Node ==> + S---D + + The source node(S) will send Gossip containing a record describing itself that + carries no IP address, since it does not accept connections; but it won't be + rejected. + */ + init_test_logging(); + let test_name = "standard_gossip_from_node_that_does_not_accept_connections_is_accepted"; + let mut src_root = make_node_record(1234, true); + let gossip_source: SocketAddr = src_root.node_addr_opt().unwrap().into(); + src_root.inner.accepts_connections = false; + src_root.metadata.node_addr_opt = None; + src_root.resign(); + let neighbor_1 = make_node_record(2345, false); + let neighbor_2 = make_node_record(3456, false); + let dest_root = make_node_record(4567, true); + let mut src_db = db_from_node(&src_root); + let mut dest_db = db_from_node(&dest_root); + // The destination Node knows the IP address of the source, even if the source doesn't + let mut src_root_with_node_addr = src_root.clone(); + src_root_with_node_addr.metadata.node_addr_opt = + Some(NodeAddr::new(&gossip_source.ip(), &[gossip_source.port()])); + dest_db.add_node(src_root_with_node_addr).unwrap(); + dest_db.add_arbitrary_full_neighbor(dest_root.public_key(), src_root.public_key()); + src_db.add_node(dest_db.root().clone()).unwrap(); + src_db.add_arbitrary_full_neighbor(src_root.public_key(), dest_root.public_key()); + src_db.add_node(neighbor_1.clone()).unwrap(); + src_db.add_arbitrary_full_neighbor(src_root.public_key(), neighbor_1.public_key()); + src_db.add_node(neighbor_2.clone()).unwrap(); + src_db.add_arbitrary_full_neighbor(src_root.public_key(), neighbor_2.public_key()); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new(test_name)); + let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); + let (cpm_recipient, _) = make_cpm_recipient(); + let mut neighborhood_metadata = make_default_neighborhood_metadata(); + neighborhood_metadata.cpm_recipient = cpm_recipient; + let gossip = GossipBuilder::new(&src_db) + .node(src_root.public_key(), false) + .node(neighbor_1.public_key(), true) + .node(neighbor_2.public_key(), true) + .build(); + + let handle_result = subject.handle( + &cryptde, + &mut dest_db, + gossip.try_into().unwrap(), + gossip_source, + neighborhood_metadata, + ); + + assert_eq!(handle_result, vec![GossipAcceptanceResult::Accepted]) + } + + #[test] + fn standard_gossip_that_does_not_describe_gossip_source_is_rejected() { + /* + Destination Node ==> + S---D + + Source Node ==> + S---D + + The source node(S) will send Gossip containing no IP address. However, we can know its + IP address from the network stack, and we'll also know its IP address from when it + debuted. Therefore, we can know to ban it. + */ + let test_name = "standard_gossip_that_does_not_describe_gossip_source_is_rejected"; + let mut src_root = make_node_record(1234, true); + let src_root_with_node_addr = src_root.clone(); + src_root.metadata.node_addr_opt = None; + src_root.inner.accepts_connections = false; + src_root.resign(); + let dest_root = make_node_record(2345, true); + let mut src_db = db_from_node(&src_root); + let mut dest_db = db_from_node(&dest_root); + dest_db.add_node(src_root_with_node_addr).unwrap(); + dest_db.add_arbitrary_full_neighbor(dest_root.public_key(), src_root.public_key()); + src_db.add_node(dest_db.root().clone()).unwrap(); + src_db.add_arbitrary_full_neighbor(src_root.public_key(), dest_root.public_key()); + let subject = + StandardGossipHandler::new(&RatePackLimits::test_default(), Logger::new(test_name)); + let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); + let gossip = GossipBuilder::new(&src_db) + .node(src_root.public_key(), false) + .build(); + let (cpm_recipient, _) = make_cpm_recipient(); + let mut neighborhood_metadata = make_default_neighborhood_metadata(); + neighborhood_metadata.cpm_recipient = cpm_recipient; + // A SocketAddr that doesn't match anything in the Standard Gossip + let malefactor_socket_addr = SocketAddr::from_str("3.4.5.6:3456").unwrap(); + + let handle_result = subject.handle( + &cryptde, + &mut dest_db, + gossip.try_into().unwrap(), + malefactor_socket_addr, + neighborhood_metadata, + ); + + assert_eq!( + handle_result, + vec![GossipAcceptanceResult::Ban(Malefactor::new( + None, + Some(malefactor_socket_addr.ip()), + None, + None, + "Node at 3.4.5.6 sent Standard gossip without a record describing itself" + .to_string() + ))] + ); } #[test] - fn standard_gossip_handler_doesnt_add_gossipping_node_if_already_max_degree() { + fn standard_gossip_that_fails_validation_is_rejected() { + /* + Destination Node ==> + S---D + + Source Node ==> + A---S---D + | + B + + The source node(S) will gossip about Nodes A and B + to the destination node(D). Node B will be dropped because its + rate pack is outside the limits. + */ + init_test_logging(); + let test_name = "standard_gossip_that_fails_validation_is_rejected"; let src_root = make_node_record(1234, true); let dest_root = make_node_record(2345, true); - let five_neighbors: Vec = (0..MAX_DEGREE as u16) - .into_iter() - .map(|index| make_node_record(5110 + index, true)) - .collect(); - let add_neighbors = |db: &mut NeighborhoodDatabase, count: usize| { - five_neighbors.iter().take(count).for_each(|node| { - db.add_node(node.clone()).unwrap(); - let root_key = db.root().public_key().clone(); - db.add_arbitrary_full_neighbor(&root_key, node.public_key()); - }); - }; let mut src_db = db_from_node(&src_root); - add_neighbors(&mut src_db, 2); - src_db.add_node(dest_root.clone()).unwrap(); - src_db.add_arbitrary_full_neighbor(five_neighbors[0].public_key(), dest_root.public_key()); + let node_a = make_node_record(5678, true); + let mut node_b = make_node_record(7777u16, true); + node_b.inner.rate_pack = RatePack::new(0, 0, 0, 0); // invalid + node_b.resign(); let mut dest_db = db_from_node(&dest_root); - let dest_cryptde = CryptDENull::from(dest_root.public_key(), TEST_DEFAULT_CHAIN); - add_neighbors(&mut dest_db, MAX_DEGREE); dest_db.add_node(src_root.clone()).unwrap(); - dest_db.add_arbitrary_full_neighbor(five_neighbors[0].public_key(), src_root.public_key()); - let gossip = GossipProducerReal::new() - .produce(&mut src_db, dest_root.public_key()) - .unwrap(); - let subject = make_subject(&dest_cryptde); + dest_db.add_arbitrary_full_neighbor(dest_root.public_key(), src_root.public_key()); + src_db.add_node(dest_db.root().clone()).unwrap(); + src_db.add_node(node_a.clone()).unwrap(); + src_db.add_node(node_b.clone()).unwrap(); + src_db.add_arbitrary_full_neighbor(src_root.public_key(), dest_root.public_key()); + src_db.add_arbitrary_half_neighbor(src_root.public_key(), &node_a.public_key()); + src_db.add_arbitrary_full_neighbor(src_root.public_key(), &node_b.public_key()); + src_db + .node_by_key_mut(src_root.public_key()) + .unwrap() + .increment_version(); + src_db.resign_node(src_root.public_key()); + let gossip = GossipBuilder::new(&src_db) + .node(src_root.public_key(), true) + .node(node_a.public_key(), false) + .node(node_b.public_key(), false) + .build(); + let rate_pack_limits = RatePackLimits::new( + RatePack::new(100, 100, 100, 100), + RatePack::new(u64::MAX, u64::MAX, u64::MAX, u64::MAX), + ); + let subject = StandardGossipHandler::new(&rate_pack_limits, Logger::new(test_name)); + let cryptde = CryptDENull::from(dest_db.root().public_key(), TEST_DEFAULT_CHAIN); + let agrs_vec: Vec = gossip.try_into().unwrap(); + let gossip_source: SocketAddr = src_root.node_addr_opt().unwrap().into(); + let (cpm_recipient, cpm_recording_arc) = make_cpm_recipient(); + let mut neighborhood_metadata = make_default_neighborhood_metadata(); + neighborhood_metadata.cpm_recipient = cpm_recipient; + let system = System::new("test"); - let result = subject.handle( + let handle_result = subject.handle( + &cryptde, &mut dest_db, - gossip.try_into().unwrap(), - src_root.node_addr_opt().clone().unwrap().into(), - make_default_neighborhood_metadata(), + agrs_vec, + gossip_source, + neighborhood_metadata, ); - assert_eq!(result, GossipAcceptanceResult::Ignored); + let analysis = format!( + "{:?}", + rate_pack_limits + .analyze(&node_b.inner.rate_pack) + .err() + .unwrap() + ); + let message = format!( + "Node {} from Standard gossip received from {:?} rejected due to rate pack limit violation: {}", + node_b.public_key(), + gossip_source.ip(), + analysis + ); + assert_eq!( + handle_result, + vec![ + GossipAcceptanceResult::Accepted, + GossipAcceptanceResult::Ban(Malefactor::new( + Some(node_b.public_key().clone()), + None, + Some(node_b.earning_wallet().clone()), + None, + message.clone() + )), + ] + ); + assert_eq!(dest_db.node_by_key(node_a.public_key()).is_some(), true); + assert_eq!(dest_db.node_by_key(node_b.public_key()).is_none(), true); // rejected + System::current().stop(); + assert_eq!(system.run(), 0); + let recording = cpm_recording_arc.lock().unwrap(); + assert_eq!(recording.len(), 0); + TestLogHandler::new().exists_log_containing(&format!("WARN: {}: {}", test_name, message)); } #[test] fn last_gossip_handler_rejects_everything() { - let subject = make_subject(CRYPTDE_PAIR.main.as_ref()); + let subject = make_subject(GA_CRYPTDE_PAIR.main.as_ref()); let reject_handler = subject.gossip_handlers.last().unwrap(); let db = make_meaningless_db(); - let (debut, _, debut_gossip_source) = make_debut(1234, Mode::Standard); - let (pass, _, pass_gossip_source) = make_pass(2345); + let (debut, _, debut_gossip_source) = + make_debut(1234, NeighborhoodModeLight::Standard.into()); + let (pass, _, pass_gossip_source) = make_pass(2345, NeighborhoodModeLight::Standard.into()); let (introduction, introduction_gossip_source) = make_introduction(3456, 4567); - let (standard_gossip, _, standard_gossip_source) = make_debut(9898, Mode::Standard); + let (standard_gossip, _, standard_gossip_source) = + make_debut(9898, NeighborhoodModeLight::Standard.into()); let debut_vec: Vec = debut.try_into().unwrap(); let pass_vec: Vec = pass.try_into().unwrap(); let introduction_vec: Vec = introduction.try_into().unwrap(); @@ -3007,31 +4289,43 @@ mod tests { assert_eq!( debut_result, - Qualification::Malformed( - "Gossip with 1 records from 1.2.3.4:1234 is unclassifiable by any qualifier" - .to_string() - ) + Qualification::Malformed(Malefactor::new( + None, + Some(debut_gossip_source.ip()), + None, + None, + "Gossip with 1 records is unclassifiable by any qualifier".to_string() + )) ); assert_eq!( pass_result, - Qualification::Malformed( - "Gossip with 1 records from 200.200.200.200:2000 is unclassifiable by any qualifier" - .to_string() - ) + Qualification::Malformed(Malefactor::new( + None, + Some(pass_gossip_source.ip()), + None, + None, + "Gossip with 1 records is unclassifiable by any qualifier".to_string() + )) ); assert_eq!( introduction_result, - Qualification::Malformed( - "Gossip with 2 records from 3.4.5.6:3456 is unclassifiable by any qualifier" - .to_string() - ) + Qualification::Malformed(Malefactor::new( + None, + Some(introduction_gossip_source.ip()), + None, + None, + "Gossip with 2 records is unclassifiable by any qualifier".to_string() + )) ); assert_eq!( standard_gossip_result, - Qualification::Malformed( - "Gossip with 1 records from 9.8.9.8:9898 is unclassifiable by any qualifier" - .to_string() - ) + Qualification::Malformed(Malefactor::new( + None, + Some(standard_gossip_source.ip()), + None, + None, + "Gossip with 1 records is unclassifiable by any qualifier".to_string() + )) ); } @@ -3056,7 +4350,7 @@ mod tests { .node(node_a.public_key(), false) .node(node_b.public_key(), false) .build(); - let subject = make_subject(CRYPTDE_PAIR.main.as_ref()); + let subject = make_subject(GA_CRYPTDE_PAIR.main.as_ref()); let result = subject.handle( &mut dest_db, @@ -3065,7 +4359,7 @@ mod tests { make_default_neighborhood_metadata(), ); - assert_eq!(GossipAcceptanceResult::Ignored, result); + assert_eq!(result, vec![]); } #[test] @@ -3073,7 +4367,8 @@ mod tests { let mut root_node = make_node_record(1234, true); let root_node_cryptde = CryptDENull::from(&root_node.public_key(), TEST_DEFAULT_CHAIN); let mut source_db = db_from_node(&root_node); - let (gossip, mut debut_node, gossip_source) = make_debut(2345, Mode::Standard); //debut node is FR + let (gossip, mut debut_node, gossip_source) = + make_debut(2345, NeighborhoodModeLight::Standard.into()); //debut node is FR let mut expected_source_db = db_from_node(&root_node); expected_source_db .add_arbitrary_half_neighbor(root_node.public_key(), debut_node.public_key()); @@ -3103,11 +4398,11 @@ mod tests { ); let after = time_t_timestamp(); - let expected_result = GossipAcceptanceResult::Reply( + let expected_result = vec![GossipAcceptanceResult::Reply( expected_gossip_response, debut_node.public_key().clone(), debut_node.node_addr_opt().unwrap(), - ); + )]; assert_eq!(result, expected_result); root_node .add_half_neighbor_key(debut_node.public_key().clone()) @@ -3139,7 +4434,8 @@ mod tests { .unwrap() .resign(); dest_db.node_by_key_mut(existing_node_key).unwrap().resign(); - let (gossip, mut debut_node, gossip_source) = make_debut(2345, Mode::Standard); + let (gossip, mut debut_node, gossip_source) = + make_debut(2345, NeighborhoodModeLight::Standard.into()); let subject = make_subject(&root_node_cryptde); let before = time_t_timestamp(); @@ -3156,12 +4452,12 @@ mod tests { .node(existing_node_key, true) .build(); assert_eq!( - GossipAcceptanceResult::Reply( + result, + vec![GossipAcceptanceResult::Reply( expected_acceptance_gossip, debut_node.public_key().clone(), debut_node.node_addr_opt().unwrap(), - ), - result + )] ); root_node .add_half_neighbor_key(debut_node.public_key().clone()) @@ -3209,7 +4505,8 @@ mod tests { .unwrap() .resign(); - let (gossip, mut debut_node, gossip_source) = make_debut(2345, Mode::Standard); + let (gossip, mut debut_node, gossip_source) = + make_debut(2345, NeighborhoodModeLight::Standard.into()); let subject = make_subject(&root_node_cryptde); let before = time_t_timestamp(); @@ -3233,16 +4530,16 @@ mod tests { let debut_node_addr = debut_node.node_addr_opt().as_ref().unwrap().clone(); assert_contains( &[ - GossipAcceptanceResult::Reply( + vec![GossipAcceptanceResult::Reply( expected_acceptance_gossip_1, debut_key.clone(), debut_node_addr.clone(), - ), - GossipAcceptanceResult::Reply( + )], + vec![GossipAcceptanceResult::Reply( expected_acceptance_gossip_2, debut_key.clone(), debut_node_addr.clone(), - ), + )], ], &result, ); @@ -3305,7 +4602,8 @@ mod tests { .unwrap() .resign(); - let (gossip, debut_node, gossip_source) = make_debut(2345, Mode::Standard); + let (gossip, debut_node, gossip_source) = + make_debut(2345, NeighborhoodModeLight::Standard.into()); let subject = make_subject(&root_node_cryptde); let result = subject.handle( @@ -3323,16 +4621,16 @@ mod tests { .build(); assert_contains( &[ - GossipAcceptanceResult::Reply( + vec![GossipAcceptanceResult::Reply( expected_acceptance_gossip_2, debut_node.public_key().clone(), debut_node.node_addr_opt().unwrap(), - ), - GossipAcceptanceResult::Reply( + )], + vec![GossipAcceptanceResult::Reply( expected_acceptance_gossip_3, debut_node.public_key().clone(), debut_node.node_addr_opt().unwrap(), - ), + )], ], &result, ); @@ -3395,7 +4693,8 @@ mod tests { .unwrap() .resign(); - let (gossip, debut_node, gossip_source) = make_debut(2345, Mode::Standard); + let (gossip, debut_node, gossip_source) = + make_debut(2345, NeighborhoodModeLight::Standard.into()); let subject = make_subject(&root_node_cryptde); let result = subject.handle( @@ -3409,12 +4708,12 @@ mod tests { .node(existing_node_5_key, true) .build(); assert_eq!( - GossipAcceptanceResult::Reply( + result, + vec![GossipAcceptanceResult::Reply( expected_acceptance_gossip, debut_node.public_key().clone(), debut_node.node_addr_opt().unwrap(), - ), - result + )], ); root_node .add_half_neighbor_key(existing_node_1_key.clone()) @@ -3452,7 +4751,7 @@ mod tests { .node(src_node.public_key(), true) .build(); let gossip_source: SocketAddr = src_node.node_addr_opt().unwrap().into(); - let subject = make_subject(CRYPTDE_PAIR.main.as_ref()); + let subject = make_subject(GA_CRYPTDE_PAIR.main.as_ref()); let result = subject.handle( &mut dest_db, @@ -3461,13 +4760,13 @@ mod tests { make_default_neighborhood_metadata(), ); - assert_eq!(GossipAcceptanceResult::Ignored, result); + assert_eq!(result, vec![]); assert_eq!( - false, dest_db .node_by_key(src_node.public_key()) .unwrap() - .has_half_neighbor(dest_node.public_key()) + .has_half_neighbor(dest_node.public_key()), + false, ); } @@ -3490,7 +4789,7 @@ mod tests { .build(); let debut_agrs = debut.try_into().unwrap(); let gossip_source: SocketAddr = src_node.node_addr_opt().unwrap().into(); - let subject = make_subject(CRYPTDE_PAIR.main.as_ref()); + let subject = make_subject(GA_CRYPTDE_PAIR.main.as_ref()); let begin_at = time_t_timestamp(); let result = subject.handle( @@ -3501,7 +4800,7 @@ mod tests { ); let end_at = time_t_timestamp(); - assert_eq!(GossipAcceptanceResult::Accepted, result); + assert_eq!(result, vec![GossipAcceptanceResult::Accepted]); let node = dest_db.node_by_key(src_node.public_key()).unwrap(); assert_eq!(node.has_half_neighbor(dest_node.public_key()), true); assert_eq!( @@ -3524,7 +4823,7 @@ mod tests { .build(); let debut_agrs = debut.try_into().unwrap(); let gossip_source = src_node.node_addr_opt().unwrap().into(); - let subject = make_subject(CRYPTDE_PAIR.main.as_ref()); + let subject = make_subject(GA_CRYPTDE_PAIR.main.as_ref()); let result = subject.handle( &mut dest_db, @@ -3541,12 +4840,15 @@ mod tests { let gnr = GossipNodeRecord::from(( root_node.inner.clone(), root_node.node_addr_opt(), - CRYPTDE_PAIR.main.as_ref(), + GA_CRYPTDE_PAIR.main.as_ref(), )); let debut_gossip = Gossip_0v1 { node_records: vec![gnr], }; - let expected = make_expected_non_introduction_debut_response(&src_node, debut_gossip); + let expected = vec![make_expected_non_introduction_debut_response( + &src_node, + debut_gossip, + )]; assert_eq!(result, expected); assert_eq!( dest_db @@ -3573,7 +4875,7 @@ mod tests { .build(); let debut_agrs = debut.try_into().unwrap(); let gossip_source = src_node.node_addr_opt().unwrap().into(); - let subject = make_subject(CRYPTDE_PAIR.main.as_ref()); + let subject = make_subject(GA_CRYPTDE_PAIR.main.as_ref()); let result = subject.handle( &mut dest_db, @@ -3582,28 +4884,79 @@ mod tests { make_default_neighborhood_metadata(), ); - let mut root_node = dest_db.root().clone(); - root_node.clear_half_neighbors(); - root_node - .add_half_neighbor_key(src_node.public_key().clone()) - .expect("expected half neighbor"); + let mut debut_back_node = dest_node.clone(); + debut_back_node + .inner + .neighbors + .insert(src_node.public_key().clone()); + debut_back_node.increment_version(); + debut_back_node.resign(); let gnr = GossipNodeRecord::from(( - root_node.inner.clone(), - root_node.node_addr_opt(), - CRYPTDE_PAIR.main.as_ref(), + debut_back_node.inner.clone(), + debut_back_node.node_addr_opt(), + GA_CRYPTDE_PAIR.main.as_ref(), )); - let debut_gossip = Gossip_0v1 { + let debut_back_gossip = Gossip_0v1 { node_records: vec![gnr], }; - let expected = make_expected_non_introduction_debut_response(&src_node, debut_gossip); - assert_eq!(result, expected); assert_eq!( - dest_db - .node_by_key(dest_node.public_key()) - .unwrap() - .has_half_neighbor(src_node.public_key()), - true, + result, + vec![GossipAcceptanceResult::Reply( + debut_back_gossip, + src_node.public_key().clone(), + src_node.node_addr_opt().unwrap().clone(), + )] + ) + } + + #[test] + fn introduction_is_impossible_if_only_candidate_does_not_route_data() { + let src_node = make_node_record(1234, true); + let src_db = db_from_node(&src_node); + let dest_node = make_node_record(2345, true); + let mut dest_db: NeighborhoodDatabase = db_from_node(&dest_node); + let nonrouting_key = &dest_db + .add_node(make_node_record_f(3456, true, true, false)) + .unwrap(); + dest_db.add_arbitrary_full_neighbor(dest_node.public_key(), nonrouting_key); + + let debut = GossipBuilder::new(&src_db) + .node(src_node.public_key(), true) + .build(); + let debut_agrs = debut.try_into().unwrap(); + let gossip_source = src_node.node_addr_opt().unwrap().into(); + let subject = make_subject(GA_CRYPTDE_PAIR.main.as_ref()); + + let result = subject.handle( + &mut dest_db, + debut_agrs, + gossip_source, + make_default_neighborhood_metadata(), ); + + let mut debut_back_node = dest_node.clone(); + debut_back_node + .inner + .neighbors + .insert(src_node.public_key().clone()); + debut_back_node.increment_version(); + debut_back_node.resign(); + let gnr = GossipNodeRecord::from(( + debut_back_node.inner.clone(), + debut_back_node.node_addr_opt(), + GA_CRYPTDE_PAIR.main.as_ref(), + )); + let debut_back_gossip = Gossip_0v1 { + node_records: vec![gnr], + }; + assert_eq!( + result, + vec![GossipAcceptanceResult::Reply( + debut_back_gossip, + src_node.public_key().clone(), + src_node.node_addr_opt().unwrap().clone(), + )] + ) } fn make_expected_non_introduction_debut_response( @@ -3619,10 +4972,11 @@ mod tests { #[test] fn introduction_gossip_handler_sends_cpm_for_neighborship_established() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = GA_CRYPTDE_PAIR.main.as_ref(); let root_node = make_node_record(1234, true); let mut db = db_from_node(&root_node); - let subject = IntroductionHandler::new(Logger::new("test")); + let subject = + IntroductionHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let (gossip, gossip_source) = make_introduction(0, 1); let (cpm_recipient, recording_arc) = make_cpm_recipient(); let mut neighborhood_metadata = make_default_neighborhood_metadata(); @@ -3664,10 +5018,14 @@ mod tests { #[test] fn pass_is_properly_handled() { // This test makes sure GossipAcceptor works correctly + init_test_logging(); + let test_name = "pass_is_properly_handled"; let root_node = make_node_record(1234, true); let mut db = db_from_node(&root_node); - let (gossip, pass_target, gossip_source) = make_pass(2345); - let subject = make_subject(CRYPTDE_PAIR.main.as_ref()); + let (gossip, pass_target, gossip_source) = + make_pass(2345, NeighborhoodModeLight::Standard.into()); + let mut subject = make_subject(GA_CRYPTDE_PAIR.main.as_ref()); + subject.logger = Logger::new(test_name); let result = subject.handle( &mut db, @@ -3681,22 +5039,30 @@ mod tests { .build(); assert_eq!( result, - GossipAcceptanceResult::Reply( + vec![GossipAcceptanceResult::Reply( expected_relay_gossip, pass_target.public_key().clone(), pass_target.node_addr_opt().unwrap(), - ) + )] ); assert_eq!(db.keys().len(), 1); + TestLogHandler::new().exists_log_containing( + format!( + "TRACE: {}: Gossip from {} did not qualify for {}: {}", + test_name, gossip_source, "DebutHandler", "Node record is not Gossip source" + ) + .as_str(), + ); } #[test] fn handles_a_new_pass_target() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = GA_CRYPTDE_PAIR.main.as_ref(); let root_node = make_node_record(1234, true); let mut db = db_from_node(&root_node); let subject = PassHandler::new(); - let (gossip, pass_target, gossip_source) = make_pass(2345); + let (gossip, pass_target, gossip_source) = + make_pass(2345, NeighborhoodModeLight::Standard.into()); let system = System::new("handles_a_new_pass_target"); let (cpm_recipient, recording_arc) = make_cpm_recipient(); let mut neighborhood_metadata = make_default_neighborhood_metadata(); @@ -3712,8 +5078,8 @@ mod tests { ); let final_timestamp = SystemTime::now(); - match result { - GossipAcceptanceResult::Reply(_, _, _) => (), + match &result[0] { + &GossipAcceptanceResult::Reply(_, _, _) => (), other => panic!( "Expected GossipAcceptanceResult::Reply but received {:?}", other @@ -3739,11 +5105,12 @@ mod tests { #[test] fn handles_pass_target_that_is_not_yet_expired() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = GA_CRYPTDE_PAIR.main.as_ref(); let root_node = make_node_record(1234, true); let mut db = db_from_node(&root_node); let subject = PassHandler::new(); - let (gossip, pass_target, gossip_source) = make_pass(2345); + let (gossip, pass_target, gossip_source) = + make_pass(2345, NeighborhoodModeLight::Standard.into()); let pass_target_ip_addr = pass_target.node_addr_opt().unwrap().ip_addr(); subject.previous_pass_targets.borrow_mut().insert( pass_target_ip_addr, @@ -3768,7 +5135,7 @@ mod tests { let final_timestamp = SystemTime::now(); System::current().stop(); assert_eq!(system.run(), 0); - assert_eq!(result, GossipAcceptanceResult::Ignored); + assert_eq!(result, vec![]); let recording = recording_arc.lock().unwrap(); let received_message: &ConnectionProgressMessage = recording.get_record(0); assert_eq!( @@ -3786,11 +5153,12 @@ mod tests { #[test] fn handles_pass_target_that_is_a_part_of_a_different_connection_progress() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = GA_CRYPTDE_PAIR.main.as_ref(); let root_node = make_node_record(1234, true); let mut db = db_from_node(&root_node); let subject = PassHandler::new(); - let (gossip, pass_target, gossip_source) = make_pass(2345); + let (gossip, pass_target, gossip_source) = + make_pass(2345, NeighborhoodModeLight::Standard.into()); let pass_target_ip_addr = pass_target.node_addr_opt().unwrap().ip_addr(); let system = System::new("handles_pass_target_that_is_a_part_of_a_different_connection_progress"); @@ -3809,7 +5177,7 @@ mod tests { System::current().stop(); assert_eq!(system.run(), 0); - assert_eq!(result, GossipAcceptanceResult::Ignored); + assert_eq!(result, vec![]); let recording = recording_arc.lock().unwrap(); let received_message: &ConnectionProgressMessage = recording.get_record(0); assert_eq!( @@ -3823,11 +5191,12 @@ mod tests { #[test] fn handles_pass_target_that_has_expired() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = GA_CRYPTDE_PAIR.main.as_ref(); let root_node = make_node_record(1234, true); let mut db = db_from_node(&root_node); let subject = PassHandler::new(); - let (gossip, pass_target, gossip_source) = make_pass(2345); + let (gossip, pass_target, gossip_source) = + make_pass(2345, NeighborhoodModeLight::Standard.into()); let (cpm_recipient, recording_arc) = make_cpm_recipient(); let mut neighborhood_metadata = make_default_neighborhood_metadata(); neighborhood_metadata.cpm_recipient = cpm_recipient; @@ -3849,8 +5218,8 @@ mod tests { ); let final_timestamp = SystemTime::now(); - match result { - GossipAcceptanceResult::Reply(_, _, _) => (), + match &result[0] { + &GossipAcceptanceResult::Reply(_, _, _) => (), other => panic!( "Expected GossipAcceptanceResult::Reply but received {:?}", other @@ -3873,7 +5242,7 @@ mod tests { } #[test] - fn standard_gossip_containing_unfamiliar_node_addrs_leads_to_them_being_ignored() { + fn standard_gossip_containing_unfamiliar_node_addrs_leads_to_them_being_banned() { /* <---- Databases before the gossip ----> @@ -3944,7 +5313,7 @@ mod tests { .node(node_e.public_key(), true) .node(node_f.public_key(), true) .build(); - let subject = make_subject(CRYPTDE_PAIR.main.as_ref()); + let subject = make_subject(GA_CRYPTDE_PAIR.main.as_ref()); let before = time_t_timestamp(); let result = subject.handle( @@ -3955,7 +5324,17 @@ mod tests { ); let after = time_t_timestamp(); - assert_eq!(GossipAcceptanceResult::Accepted, result); + assert_eq!(result, vec![ + GossipAcceptanceResult::Accepted, + GossipAcceptanceResult::Ban(Malefactor::from(( + &node_a, + "Node AgMEBQ at 2.3.4.5 sent Standard gossip that contained an IP address for victim Node BgcICQ that we should not have known".to_string()) + )), + GossipAcceptanceResult::Ban(Malefactor::from(( + &node_a, + "Node AgMEBQ at 2.3.4.5 sent Standard gossip that contained an IP address for victim Node BwgJAA that we should not have known".to_string()) + )), + ]); let mut expected_dest_db = src_db.clone(); expected_dest_db.remove_arbitrary_half_neighbor(node_e.public_key(), node_a.public_key()); expected_dest_db.remove_arbitrary_half_neighbor(node_f.public_key(), node_a.public_key()); @@ -4008,6 +5387,108 @@ mod tests { assert_eq!(dest_db.node_by_key(node_f.public_key()), None); } + struct MalformedHandler { + malefactor: Malefactor, + } + + impl NamedType for MalformedHandler { + fn type_name(&self) -> &'static str { + "MalformedHandler" + } + } + impl GossipHandler for MalformedHandler { + fn qualifies( + &self, + _database: &NeighborhoodDatabase, + _agrs: &[AccessibleGossipRecord], + _gossip_source: SocketAddr, + ) -> Qualification { + Qualification::Malformed(self.malefactor.clone()) + } + + fn handle( + &self, + _cryptde: &dyn CryptDE, + _database: &mut NeighborhoodDatabase, + _agrs: Vec, + _gossip_source: SocketAddr, + _neighborhood_metadata: NeighborhoodMetadata, + ) -> Vec { + unimplemented!("Should never be called") + } + + fn as_any(&self) -> &dyn Any { + self + } + } + + impl MalformedHandler { + fn new(malefactor: &Malefactor) -> Self { + MalformedHandler { + malefactor: malefactor.clone(), + } + } + } + + #[test] + fn qualification_of_malformed_with_agr_produces_malefactor_ban() { + let dest_root = make_node_record(1234, true); + let mut dest_db = db_from_node(&dest_root); // irrelevant + let src_root = make_node_record(2345, true); + let src_db = db_from_node(&src_root); + let gossip = GossipBuilder::new(&src_db) + .node(src_root.public_key(), true) + .build(); + let malefactor = Malefactor::new( + Some(src_root.inner.public_key.clone()), + Some(src_root.node_addr_opt().unwrap().ip_addr()), + Some(src_root.earning_wallet()), + Some(Wallet::new("consuming_wallet")), + "Malformed for test".to_string(), + ); + let malformed_handler = Box::new(MalformedHandler::new(&malefactor)); + let mut subject = make_subject(GA_CRYPTDE_PAIR.main.as_ref()); + subject.gossip_handlers = vec![malformed_handler]; + + let result = subject.handle( + &mut dest_db, + gossip.try_into().unwrap(), + src_root.node_addr_opt().unwrap().into(), + make_default_neighborhood_metadata(), + ); + + assert_eq!(result, vec![GossipAcceptanceResult::Ban(malefactor)]); + } + + #[test] + fn qualification_of_malformed_without_agr_produces_malefactor_ban() { + let dest_root = make_node_record(1234, true); + let mut dest_db = db_from_node(&dest_root); // irrelevant + let src_root = make_node_record(2345, true); + let src_db = db_from_node(&src_root); + let gossip = GossipBuilder::new(&src_db).build(); + let malefactor = Malefactor::new( + Some(src_root.inner.public_key.clone()), + Some(src_root.node_addr_opt().unwrap().ip_addr()), + Some(src_root.earning_wallet()), + Some(Wallet::new("consuming_wallet")), + "Malformed for test".to_string(), + ); + let malformed_handler = Box::new(MalformedHandler::new(&malefactor)); + let mut subject = make_subject(GA_CRYPTDE_PAIR.main.as_ref()); + subject.gossip_handlers = vec![malformed_handler]; + let gossip_source = SocketAddr::from_str("5.5.5.5:5555").unwrap(); // not in Gossip + + let result = subject.handle( + &mut dest_db, + gossip.try_into().unwrap(), + gossip_source, // not in Gossip + make_default_neighborhood_metadata(), + ); + + assert_eq!(result, vec![GossipAcceptanceResult::Ban(malefactor)]); + } + fn fix_nodes_last_updates( expected_db: &mut NeighborhoodDatabase, dest_db: &NeighborhoodDatabase, @@ -4059,9 +5540,8 @@ mod tests { let mut dest_db = db_from_node(&dest_node); let src_node = make_node_record(2345, true); let mut src_db = db_from_node(&src_node); - let third_node = make_node_record(3456, true); - let disconnected_node = make_node_record(4567, true); // Why does this have an Ip Address? - // These are only half neighbors. Will they be ignored in degree calculation? + let third_node = make_node_record(3456, false); + let disconnected_node = make_node_record(4567, false); for idx in 0..MAX_DEGREE { let failed_node_key = &dest_db .add_node(make_node_record(4000 + idx as u16, true)) @@ -4083,7 +5563,7 @@ mod tests { .node_by_key_mut(third_node.public_key()) .unwrap() .increment_version(); - resign_nodes(&mut src_db, vec![&src_node, &dest_node, &third_node]); + resign_nodes(&mut src_db, vec![&src_node, &third_node]); let gossip = GossipBuilder::new(&src_db) .node(src_node.public_key(), true) .node(third_node.public_key(), true) @@ -4116,7 +5596,7 @@ mod tests { .unwrap(); dest_node_mut.increment_version(); dest_node_mut.resign(); - assert_eq!(result, GossipAcceptanceResult::Accepted); + assert_eq!(result, vec![GossipAcceptanceResult::Accepted]); fix_nodes_last_updates(&mut expected_dest_db, &dest_db); assert_node_records_eq( dest_db.node_by_key(third_node.public_key()).unwrap(), @@ -4176,9 +5656,9 @@ mod tests { .node(current_node.public_key(), false) .node(obsolete_node.public_key(), false) .build(); - let subject = make_subject(CRYPTDE_PAIR.main.as_ref()); + let subject = make_subject(GA_CRYPTDE_PAIR.main.as_ref()); let original_dest_db = dest_db.clone(); - let before = time_t_timestamp(); + let before = time_t_timestamp() - 3600; let result = subject.handle( &mut dest_db, @@ -4187,8 +5667,8 @@ mod tests { make_default_neighborhood_metadata(), ); - let after = time_t_timestamp(); - assert_eq!(result, GossipAcceptanceResult::Ignored); + let after = time_t_timestamp() + 3600; + assert_eq!(result, vec![]); assert_node_records_eq( dest_db.node_by_key(dest_root.public_key()).unwrap(), original_dest_db @@ -4281,6 +5761,59 @@ mod tests { ); } + #[test] + fn validate_new_version_is_okay_with_valid_rate_pack() { + let test_name = "validate_new_version_is_okay_with_valid_rate_pack"; + let mut node = make_node_record(1234, true); + node.inner.rate_pack = DEFAULT_RATE_PACK.clone(); + let agr = AccessibleGossipRecord::from(&node); + + let result = GossipAcceptorReal::validate_incoming_agr( + &agr, + "description".to_string(), + &DEFAULT_RATE_PACK_LIMITS, + &Logger::new(test_name), + ); + + assert_eq!(result, Ok(())); + } + + #[test] + fn validate_new_version_doesnt_like_invalid_rate_pack_if_routes_data() { + let test_name = "validate_new_version_doesnt_like_invalid_rate_pack_if_routes_data"; + let mut node = make_node_record(1234, true); + node.inner.routes_data = true; + node.inner.rate_pack = ZERO_RATE_PACK.clone(); + let agr = AccessibleGossipRecord::from(&node); + + let result = GossipAcceptorReal::validate_incoming_agr( + &agr, + "description".to_string(), + &DEFAULT_RATE_PACK_LIMITS, + &Logger::new(test_name), + ); + + assert_eq!(result, Err("description rejected due to rate pack limit violation: ConfiguratorError { param_errors: [ParamError { parameter: \"rate-pack\", reason: \"Value of routing_byte_rate (0) is below the minimum allowed (100)\" }, ParamError { parameter: \"rate-pack\", reason: \"Value of routing_service_rate (0) is below the minimum allowed (100)\" }, ParamError { parameter: \"rate-pack\", reason: \"Value of exit_byte_rate (0) is below the minimum allowed (100)\" }, ParamError { parameter: \"rate-pack\", reason: \"Value of exit_service_rate (0) is below the minimum allowed (100)\" }] }".to_string())); + } + + #[test] + fn validate_new_version_is_okay_with_invalid_rate_pack_if_doesnt_route_data() { + let test_name = "validate_new_version_is_okay_with_invalid_rate_pack_if_doesnt_route_data"; + let mut node = make_node_record(1234, true); + node.inner.routes_data = false; + node.inner.rate_pack = ZERO_RATE_PACK.clone(); + let agr = AccessibleGossipRecord::from(&node); + + let result = GossipAcceptorReal::validate_incoming_agr( + &agr, + "description".to_string(), + &DEFAULT_RATE_PACK_LIMITS, + &Logger::new(test_name), + ); + + assert_eq!(result, Ok(())); + } + #[test] fn find_more_appropriate_neighbor_rejects_twos_and_finds_three() { let root_node = make_node_record(1234, true); @@ -4296,7 +5829,7 @@ mod tests { db.add_arbitrary_full_neighbor(root_node.public_key(), other_neighbor_3_key); db.add_arbitrary_full_neighbor(less_connected_neighbor_key, other_neighbor_1_key); db.add_arbitrary_full_neighbor(less_connected_neighbor_key, other_neighbor_2_key); - let subject = DebutHandler::new(Logger::new("test")); + let subject = DebutHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let excluded = AccessibleGossipRecord::from((&db, excluded_key, true)); let result = subject.find_more_appropriate_neighbor(&db, &excluded); @@ -4320,7 +5853,7 @@ mod tests { db.add_arbitrary_full_neighbor(less_connected_neighbor_key, other_neighbor_1_key); db.add_arbitrary_full_neighbor(less_connected_neighbor_key, other_neighbor_2_key); db.add_arbitrary_full_neighbor(less_connected_neighbor_key, other_neighbor_3_key); - let subject = DebutHandler::new(Logger::new("test")); + let subject = DebutHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let excluded = AccessibleGossipRecord::from((&db, excluded_key, true)); let result = subject.find_more_appropriate_neighbor(&db, &excluded); @@ -4347,7 +5880,7 @@ mod tests { db.add_arbitrary_full_neighbor(root_node.public_key(), other_neighbor_4_key); db.add_arbitrary_full_neighbor(less_connected_neighbor_key, other_neighbor_1_key); db.add_arbitrary_full_neighbor(less_connected_neighbor_key, other_neighbor_2_key); - let subject = DebutHandler::new(Logger::new("test")); + let subject = DebutHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let excluded = AccessibleGossipRecord::from((&db, excluded_key, true)); let result = subject.find_more_appropriate_neighbor(&db, &excluded); @@ -4369,7 +5902,7 @@ mod tests { db.add_arbitrary_full_neighbor(root_node.public_key(), other_neighbor_3_key); db.add_arbitrary_full_neighbor(less_connected_neighbor_key, other_neighbor_1_key); db.add_arbitrary_full_neighbor(less_connected_neighbor_key, other_neighbor_2_key); - let subject = DebutHandler::new(Logger::new("test")); + let subject = DebutHandler::new(&RatePackLimits::test_default(), Logger::new("test")); let less_connected_neighbor_agr = AccessibleGossipRecord::from((&db, less_connected_neighbor_key, true)); @@ -4460,6 +5993,40 @@ mod tests { ); } + #[test] + fn should_not_make_another_introduction_says_false_for_0_routing_neighbors() { + let root_node = make_node_record(1234, true); + let mut db = db_from_node(&root_node); + // Two neighbors, but they don't route data, so they don't count. + let mut no_routing_1 = make_node_record(2345, true); + no_routing_1.inner.routes_data = false; + db.add_node(no_routing_1.clone()).unwrap(); + db.add_arbitrary_full_neighbor(&root_node.public_key(), &no_routing_1.public_key()); + let mut no_routing_2 = make_node_record(3456, true); + no_routing_2.inner.routes_data = false; + db.add_node(no_routing_2.clone()).unwrap(); + db.add_arbitrary_full_neighbor(&root_node.public_key(), &no_routing_2.public_key()); + let agr = AccessibleGossipRecord::from(db.root()); + + let result = DebutHandler::should_not_make_another_introduction(&db, &agr); + + assert_eq!(result, false); + } + + #[test] + fn should_not_make_another_introduction_says_true_for_1_routing_neighbor() { + let root_node = make_node_record(1234, true); + let mut db = db_from_node(&root_node); + let neighbor = make_node_record(2345, true); + db.add_node(neighbor.clone()).unwrap(); + db.add_arbitrary_full_neighbor(&root_node.public_key(), &neighbor.public_key()); + let agr = AccessibleGossipRecord::from(db.root()); + + let result = DebutHandler::should_not_make_another_introduction(&db, &agr); + + assert_eq!(result, true); + } + fn resign_nodes(db: &mut NeighborhoodDatabase, nodes: Vec<&NodeRecord>) { nodes .into_iter() @@ -4475,8 +6042,8 @@ mod tests { (gossip, debut_node, gossip_source) } - fn make_pass(n: u16) -> (Gossip_0v1, NodeRecord, SocketAddr) { - let (gossip, debut_node) = make_single_node_gossip(n, Mode::Standard); + fn make_pass(n: u16, mode: Mode) -> (Gossip_0v1, NodeRecord, SocketAddr) { + let (gossip, debut_node) = make_single_node_gossip(n, mode); ( gossip, debut_node, @@ -4496,10 +6063,10 @@ mod tests { fn make_introduction(introducer_n: u16, introducee_n: u16) -> (Gossip_0v1, SocketAddr) { let mut introducer_node: NodeRecord = make_node_record(introducer_n, true); - adjust_for_mode(&mut introducer_node, Mode::Standard); + adjust_for_mode(&mut introducer_node, NeighborhoodModeLight::Standard.into()); introducer_node.set_version(10); let mut introducee_node: NodeRecord = make_node_record(introducee_n, true); - adjust_for_mode(&mut introducee_node, Mode::Standard); + adjust_for_mode(&mut introducee_node, NeighborhoodModeLight::Standard.into()); introducee_node.set_version(10); let introducer_key = introducer_node.public_key().clone(); let introducee_key = introducee_node.public_key().clone(); @@ -4527,22 +6094,19 @@ mod tests { } fn adjust_for_mode(node: &mut NodeRecord, mode: Mode) { - match mode { - Mode::Standard => { - assert!(node.node_addr_opt().is_some()); - node.inner.accepts_connections = true; - node.inner.routes_data = true; - } - Mode::OriginateOnly => { - node.metadata.node_addr_opt = None; - node.inner.accepts_connections = false; - node.inner.routes_data = true; - } + if mode.accepts_connections && mode.routes_data { + assert!(node.node_addr_opt().is_some()); } + node.inner.accepts_connections = mode.accepts_connections; + node.inner.routes_data = mode.routes_data; } fn make_subject(crypt_de: &dyn CryptDE) -> GossipAcceptorReal { - GossipAcceptorReal::new(crypt_de.dup()) + GossipAcceptorReal::new( + crypt_de.dup(), + &PersistentConfigurationMock::new() + .rate_pack_limits_result(Ok(RatePackLimits::test_default())), + ) } fn assert_node_records_eq(actual: &NodeRecord, expected: &NodeRecord, before: u32, after: u32) { diff --git a/node/src/neighborhood/malefactor.rs b/node/src/neighborhood/malefactor.rs new file mode 100644 index 000000000..fc7e65f43 --- /dev/null +++ b/node/src/neighborhood/malefactor.rs @@ -0,0 +1,449 @@ +use crate::neighborhood::gossip::AccessibleGossipRecord; +use crate::neighborhood::node_record::NodeRecord; +use crate::sub_lib::cryptde::PublicKey; +use crate::sub_lib::wallet::Wallet; +use lazy_static::lazy_static; +use std::fmt::{Display, Formatter}; +use std::net::IpAddr; +use time::{OffsetDateTime, PrimitiveDateTime}; + +lazy_static! { + pub static ref FUDGE_FACTOR: time::Duration = time::Duration::seconds(1); +} + +#[derive(Clone, Debug, Eq)] +pub struct Malefactor { + pub public_key_opt: Option, + pub ip_address_opt: Option, + pub earning_wallet_opt: Option, + pub consuming_wallet_opt: Option, + pub timestamp: PrimitiveDateTime, + pub reason: String, +} + +impl Display for Malefactor { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Malefactor{}{}{} detected at {}: {}", + match &self.public_key_opt { + Some(pk) => format!(" {}", pk), + None => "".to_string(), + }, + match &self.ip_address_opt { + Some(ip) => format!(" at {}", ip), + None => "".to_string(), + }, + match &self.earning_wallet_opt { + Some(wallet) => format!(" with earning wallet {}", wallet), + None => "".to_string(), + } + &match ( + &self.earning_wallet_opt.is_some(), + &self.consuming_wallet_opt + ) { + (true, Some(wallet)) => format!(", consuming wallet {}", wallet), + (false, Some(wallet)) => format!(" with consuming wallet {}", wallet), + (_, None) => "".to_string(), + }, + self.timestamp, + self.reason + ) + } +} + +impl PartialEq for Malefactor { + // Logic behind this custom implementation: + // The only place we will ever compare Malefactors for equality is in tests. In tests, + // the Malefactor that is generated in the production code and the Malefactor that is used + // for assertion will frequently be created a few microseconds apart, and therefore their + // timestamps will not be identical and the equality assertion will fail, even when the two + // are logically the same. Therefore, this custom implementation will consider two Malefactors + // equal if their timestamps are within FUDGE_FACTOR of each other and all their other fields are + // equal. + fn eq(&self, other: &Self) -> bool { + let equal = self.public_key_opt == other.public_key_opt + && self.ip_address_opt == other.ip_address_opt + && self.earning_wallet_opt == other.earning_wallet_opt + && self.consuming_wallet_opt == other.consuming_wallet_opt + && self.reason == other.reason; + let plus_fudge_factor = self.timestamp.saturating_add(*FUDGE_FACTOR); + let minus_fudge_factor = self.timestamp.saturating_sub(*FUDGE_FACTOR); + equal && (other.timestamp >= minus_fudge_factor && other.timestamp <= plus_fudge_factor) + } +} + +impl From<(&NodeRecord, String)> for Malefactor { + fn from(pair: (&NodeRecord, String)) -> Self { + let (node_record, reason) = pair; + Self::new( + Some(node_record.public_key().clone()), + node_record + .metadata + .node_addr_opt + .as_ref() + .map(|na| na.ip_addr()), + Some(node_record.inner.earning_wallet.clone()), + None, + reason, + ) + } +} + +impl From<(&AccessibleGossipRecord, String)> for Malefactor { + fn from(pair: (&AccessibleGossipRecord, String)) -> Self { + let (agr, reason) = pair; + Self::new( + Some(agr.inner.public_key.clone()), + agr.node_addr_opt.as_ref().map(|na| na.ip_addr()), + Some(agr.inner.earning_wallet.clone()), + None, + reason, + ) + } +} + +impl Malefactor { + pub fn new( + public_key_opt: Option, + ip_address_opt: Option, + earning_wallet_opt: Option, + consuming_wallet_opt: Option, + reason: String, + ) -> Self { + if public_key_opt.is_none() + && ip_address_opt.is_none() + && earning_wallet_opt.is_none() + && consuming_wallet_opt.is_none() + { + panic!("Malefactor must have at least one identifying attribute"); + } + Self { + public_key_opt, + ip_address_opt, + earning_wallet_opt, + consuming_wallet_opt, + timestamp: Self::timestamp(), + reason, + } + } + + fn timestamp() -> PrimitiveDateTime { + let odt = OffsetDateTime::now_local().unwrap_or_else(|_| OffsetDateTime::now_utc()); + PrimitiveDateTime::new(odt.date(), odt.time()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::str::FromStr; + + #[should_panic(expected = "Malefactor must have at least one identifying attribute")] + #[test] + fn doesnt_tolerate_all_nones() { + let _ = Malefactor::new(None, None, None, None, "Bad Smell".to_string()); + } + + #[test] + fn timestamps_properly() { + let before = Malefactor::timestamp(); + + let malefactor = Malefactor::new( + Some(PublicKey::from(&b"Booga"[..])), + None, + None, + None, + "Bad Smell".to_string(), + ); + + let after = Malefactor::timestamp(); + assert!(malefactor.timestamp >= before); + assert!(malefactor.timestamp <= after); + } + + #[test] + fn displays_public_key() { + let public_key = PublicKey::from(&b"Booga"[..]); + let malefactor = Malefactor { + public_key_opt: Some(public_key), + ip_address_opt: None, + earning_wallet_opt: None, + consuming_wallet_opt: None, + timestamp: PrimitiveDateTime::new( + time::Date::from_calendar_date(2024, time::Month::June, 1).unwrap(), + time::Time::from_hms(12, 34, 56).unwrap(), + ), + reason: "Bad Smell".to_string(), + }; + + let string = format!("{}", malefactor); + + assert_eq!( + string, + "Malefactor Qm9vZ2E detected at 2024-06-01 12:34:56.0: Bad Smell".to_string() + ); + } + + #[test] + fn displays_ip_address() { + let ip_address = IpAddr::from_str("12.34.56.78").unwrap(); + let malefactor = Malefactor { + public_key_opt: None, + ip_address_opt: Some(ip_address), + earning_wallet_opt: None, + consuming_wallet_opt: None, + timestamp: PrimitiveDateTime::new( + time::Date::from_calendar_date(2024, time::Month::June, 1).unwrap(), + time::Time::from_hms(12, 34, 56).unwrap(), + ), + reason: "Bad Smell".to_string(), + }; + + let string = format!("{}", malefactor); + + assert_eq!( + string, + "Malefactor at 12.34.56.78 detected at 2024-06-01 12:34:56.0: Bad Smell".to_string() + ); + } + + #[test] + fn displays_earning_wallet() { + let earning_wallet = + Wallet::from_str("0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee").unwrap(); + let malefactor = Malefactor { + public_key_opt: None, + ip_address_opt: None, + earning_wallet_opt: Some(earning_wallet), + consuming_wallet_opt: None, + timestamp: PrimitiveDateTime::new( + time::Date::from_calendar_date(2024, time::Month::June, 1).unwrap(), + time::Time::from_hms(12, 34, 56).unwrap(), + ), + reason: "Bad Smell".to_string(), + }; + + let string = format!("{}", malefactor); + + assert_eq!( + string, + "Malefactor with earning wallet 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee detected at 2024-06-01 12:34:56.0: Bad Smell".to_string() + ); + } + + #[test] + fn displays_consuming_wallet() { + let consuming_wallet = + Wallet::from_str("0xcccccccccccccccccccccccccccccccccccccccc").unwrap(); + let malefactor = Malefactor { + public_key_opt: None, + ip_address_opt: None, + earning_wallet_opt: None, + consuming_wallet_opt: Some(consuming_wallet), + timestamp: PrimitiveDateTime::new( + time::Date::from_calendar_date(2024, time::Month::June, 1).unwrap(), + time::Time::from_hms(12, 34, 56).unwrap(), + ), + reason: "Bad Smell".to_string(), + }; + + let string = format!("{}", malefactor); + + assert_eq!( + string, + "Malefactor with consuming wallet 0xcccccccccccccccccccccccccccccccccccccccc detected at 2024-06-01 12:34:56.0: Bad Smell".to_string() + ); + } + + #[test] + fn displays_all_fields() { + let public_key = PublicKey::from(&b"Booga"[..]); + let ip_address = IpAddr::from_str("12.34.56.78").unwrap(); + let earning_wallet = + Wallet::from_str("0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee").unwrap(); + let consuming_wallet = + Wallet::from_str("0xcccccccccccccccccccccccccccccccccccccccc").unwrap(); + let malefactor = Malefactor { + public_key_opt: Some(public_key), + ip_address_opt: Some(ip_address), + earning_wallet_opt: Some(earning_wallet), + consuming_wallet_opt: Some(consuming_wallet), + timestamp: PrimitiveDateTime::new( + time::Date::from_calendar_date(2024, time::Month::June, 1).unwrap(), + time::Time::from_hms(12, 34, 56).unwrap(), + ), + reason: "Bad Smell".to_string(), + }; + + let string = format!("{}", malefactor); + + assert_eq!( + string, + "Malefactor Qm9vZ2E at 12.34.56.78 with earning wallet 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee, consuming wallet 0xcccccccccccccccccccccccccccccccccccccccc detected at 2024-06-01 12:34:56.0: Bad Smell".to_string() + ); + } + + #[test] + fn eq_works_for_equal_timestamps() { + let public_key = PublicKey::from(&b"Booga"[..]); + let ip_address = IpAddr::from_str("12.34.56.78").unwrap(); + let earning_wallet = + Wallet::from_str("0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee").unwrap(); + let consuming_wallet = + Wallet::from_str("0xcccccccccccccccccccccccccccccccccccccccc").unwrap(); + let timestamp = PrimitiveDateTime::new( + time::Date::from_calendar_date(2024, time::Month::June, 1).unwrap(), + time::Time::from_hms(12, 34, 56).unwrap(), + ); + let reason = "Bad Smell".to_string(); + let a = Malefactor { + public_key_opt: Some(public_key.clone()), + ip_address_opt: Some(ip_address.clone()), + earning_wallet_opt: Some(earning_wallet.clone()), + consuming_wallet_opt: Some(consuming_wallet.clone()), + timestamp, + reason: reason.clone(), + }; + let b = Malefactor { + public_key_opt: Some(public_key.clone()), + ip_address_opt: Some(ip_address.clone()), + earning_wallet_opt: Some(earning_wallet.clone()), + consuming_wallet_opt: Some(consuming_wallet.clone()), + timestamp, + reason: reason.clone(), + }; + let c = Malefactor { + public_key_opt: None, + ip_address_opt: Some(ip_address.clone()), + earning_wallet_opt: Some(earning_wallet.clone()), + consuming_wallet_opt: Some(consuming_wallet.clone()), + timestamp, + reason: reason.clone(), + }; + let d = Malefactor { + public_key_opt: Some(public_key.clone()), + ip_address_opt: None, + earning_wallet_opt: Some(earning_wallet.clone()), + consuming_wallet_opt: Some(consuming_wallet.clone()), + timestamp, + reason: reason.clone(), + }; + let e = Malefactor { + public_key_opt: Some(public_key.clone()), + ip_address_opt: Some(ip_address.clone()), + earning_wallet_opt: None, + consuming_wallet_opt: Some(consuming_wallet.clone()), + timestamp, + reason: reason.clone(), + }; + let f = Malefactor { + public_key_opt: Some(public_key.clone()), + ip_address_opt: Some(ip_address.clone()), + earning_wallet_opt: Some(earning_wallet.clone()), + consuming_wallet_opt: None, + timestamp, + reason: reason.clone(), + }; + let g = Malefactor { + public_key_opt: Some(public_key.clone()), + ip_address_opt: Some(ip_address.clone()), + earning_wallet_opt: Some(earning_wallet.clone()), + consuming_wallet_opt: Some(consuming_wallet.clone()), + timestamp, + reason: "".to_string(), + }; + + assert_eq!(a, a); + assert_eq!(a, b); + assert_ne!(a, c); + assert_ne!(a, d); + assert_ne!(a, e); + assert_ne!(a, f); + assert_ne!(a, g); + } + + #[test] + fn eq_says_true_for_timestamps_exactly_one_second_apart() { + let public_key = PublicKey::from(&b"Booga"[..]); + let ip_address = IpAddr::from_str("12.34.56.78").unwrap(); + let earning_wallet = + Wallet::from_str("0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee").unwrap(); + let consuming_wallet = + Wallet::from_str("0xcccccccccccccccccccccccccccccccccccccccc").unwrap(); + let timestamp_early = PrimitiveDateTime::new( + time::Date::from_calendar_date(2024, time::Month::June, 1).unwrap(), + time::Time::from_hms(12, 34, 56).unwrap(), + ); + let timestamp_late = timestamp_early.saturating_add(FUDGE_FACTOR.clone()); + let reason = "Bad Smell".to_string(); + let a = Malefactor { + public_key_opt: Some(public_key.clone()), + ip_address_opt: Some(ip_address.clone()), + earning_wallet_opt: Some(earning_wallet.clone()), + consuming_wallet_opt: Some(consuming_wallet.clone()), + timestamp: timestamp_early, + reason: reason.clone(), + }; + let b = Malefactor { + public_key_opt: Some(public_key.clone()), + ip_address_opt: Some(ip_address.clone()), + earning_wallet_opt: Some(earning_wallet.clone()), + consuming_wallet_opt: Some(consuming_wallet.clone()), + timestamp: timestamp_late, + reason: reason.clone(), + }; + + assert_eq!(a, b); + assert_eq!(b, a); + } + + #[test] + fn eq_says_false_for_timestamps_more_than_one_second_apart() { + let public_key = PublicKey::from(&b"Booga"[..]); + let ip_address = IpAddr::from_str("12.34.56.78").unwrap(); + let earning_wallet = + Wallet::from_str("0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee").unwrap(); + let consuming_wallet = + Wallet::from_str("0xcccccccccccccccccccccccccccccccccccccccc").unwrap(); + let timestamp_early = PrimitiveDateTime::new( + time::Date::from_calendar_date(2024, time::Month::June, 1).unwrap(), + time::Time::from_hms(12, 34, 56).unwrap(), + ); + let timestamp_middle = timestamp_early + .saturating_add(FUDGE_FACTOR.saturating_add(time::Duration::milliseconds(1))); + let timestamp_late = timestamp_middle + .saturating_add(FUDGE_FACTOR.saturating_add(time::Duration::milliseconds(1))); + let reason = "Bad Smell".to_string(); + let a = Malefactor { + public_key_opt: Some(public_key.clone()), + ip_address_opt: Some(ip_address.clone()), + earning_wallet_opt: Some(earning_wallet.clone()), + consuming_wallet_opt: Some(consuming_wallet.clone()), + timestamp: timestamp_early, + reason: reason.clone(), + }; + let b = Malefactor { + public_key_opt: Some(public_key.clone()), + ip_address_opt: Some(ip_address.clone()), + earning_wallet_opt: Some(earning_wallet.clone()), + consuming_wallet_opt: Some(consuming_wallet.clone()), + timestamp: timestamp_middle, + reason: reason.clone(), + }; + let c = Malefactor { + public_key_opt: Some(public_key.clone()), + ip_address_opt: Some(ip_address.clone()), + earning_wallet_opt: Some(earning_wallet.clone()), + consuming_wallet_opt: Some(consuming_wallet.clone()), + timestamp: timestamp_late, + reason: reason.clone(), + }; + + assert_ne!(a, b); + assert_ne!(b, a); + assert_ne!(b, c); + assert_ne!(c, b); + assert_ne!(a, c); + assert_ne!(c, a); + } +} diff --git a/node/src/neighborhood/mod.rs b/node/src/neighborhood/mod.rs index 9c685c0af..62c050e7c 100644 --- a/node/src/neighborhood/mod.rs +++ b/node/src/neighborhood/mod.rs @@ -4,19 +4,19 @@ pub mod dot_graph; pub mod gossip; pub mod gossip_acceptor; pub mod gossip_producer; +mod malefactor; pub mod neighborhood_database; pub mod node_location; pub mod node_record; pub mod overall_connection_status; use crate::bootstrapper::{BootstrapperConfig, CryptDEPair}; -use crate::database::db_initializer::DbInitializationConfig; -use crate::database::db_initializer::{DbInitializer, DbInitializerReal}; use crate::db_config::persistent_configuration::{ - PersistentConfigError, PersistentConfiguration, PersistentConfigurationReal, + PersistentConfigError, PersistentConfiguration, PersistentConfigurationFactory, + PersistentConfigurationFactoryReal, PersistentConfigurationInvalid, }; use crate::neighborhood::gossip::{AccessibleGossipRecord, DotGossipEndpoint, Gossip_0v1}; -use crate::neighborhood::gossip_acceptor::GossipAcceptanceResult; +use crate::neighborhood::gossip_acceptor::{GossipAcceptanceResult, GossipAcceptorInvalid}; use crate::neighborhood::node_location::get_node_location; use crate::neighborhood::overall_connection_status::{ OverallConnectionStage, OverallConnectionStatus, @@ -27,12 +27,15 @@ use crate::sub_lib::cryptde::PublicKey; use crate::sub_lib::dispatcher::{Component, StreamShutdownMsg}; use crate::sub_lib::hopper::{ExpiredCoresPackage, NoLookupIncipientCoresPackage}; use crate::sub_lib::hopper::{IncipientCoresPackage, MessageType}; +use crate::sub_lib::host::Host; +use crate::sub_lib::neighborhood::ConnectionProgressEvent; +use crate::sub_lib::neighborhood::ExpectedService::{Exit, Nothing, Routing}; +use crate::sub_lib::neighborhood::ExpectedServices::RoundTrip; use crate::sub_lib::neighborhood::RouteQueryResponse; use crate::sub_lib::neighborhood::UpdateNodeRecordMetadataMessage; use crate::sub_lib::neighborhood::{AskAboutDebutGossipMessage, NodeDescriptor}; use crate::sub_lib::neighborhood::{ConfigChange, RemoveNeighborMessage}; use crate::sub_lib::neighborhood::{ConfigChangeMsg, RouteQueryMessage}; -use crate::sub_lib::neighborhood::{ConnectionProgressEvent, ExpectedServices}; use crate::sub_lib::neighborhood::{ConnectionProgressMessage, ExpectedService}; use crate::sub_lib::neighborhood::{DispatcherNodeQueryMessage, GossipFailure_0v1}; use crate::sub_lib::neighborhood::{Hops, NeighborhoodMetadata, NodeQueryResponseMetadata}; @@ -43,9 +46,7 @@ use crate::sub_lib::peer_actors::{BindMessage, NewPublicIp, StartMessage}; use crate::sub_lib::route::Route; use crate::sub_lib::route::RouteSegment; use crate::sub_lib::stream_handler_pool::DispatcherNodeQueryResponse; -use crate::sub_lib::utils::{ - db_connection_launch_panic, handle_ui_crash_request, NODE_MAILBOX_CAPACITY, -}; +use crate::sub_lib::utils::{handle_ui_crash_request, NODE_MAILBOX_CAPACITY}; use crate::sub_lib::versioned_data::VersionedData; use crate::sub_lib::wallet::Wallet; use actix::Context; @@ -79,7 +80,6 @@ use std::collections::HashSet; use std::convert::TryFrom; use std::fmt::Debug; use std::net::{IpAddr, SocketAddr}; -use std::path::PathBuf; use std::string::ToString; pub const CRASH_KEY: &str = "NEIGHBORHOOD"; @@ -105,12 +105,11 @@ pub struct Neighborhood { mode: NeighborhoodModeLight, min_hops: Hops, db_patch_size: u8, - next_return_route_id: u32, overall_connection_status: OverallConnectionStatus, chain: Chain, crashable: bool, - data_directory: PathBuf, - persistent_config_opt: Option>, + persistent_config_factory: Box, + persistent_config: Box, db_password_opt: Option, logger: Logger, tools: NeighborhoodTools, @@ -423,19 +422,20 @@ impl Neighborhood { hopper_no_lookup_opt: None, connected_signal_opt: None, node_to_ui_recipient_opt: None, - gossip_acceptor: Box::new(GossipAcceptorReal::new(cryptde_pair.main.dup())), + gossip_acceptor: Box::new(GossipAcceptorInvalid::new()), gossip_producer: Box::new(GossipProducerReal::new()), neighborhood_database, consuming_wallet_opt: config.consuming_wallet_opt.clone(), mode, min_hops, db_patch_size, - next_return_route_id: 0, overall_connection_status, chain: config.blockchain_bridge_config.chain, crashable: config.crash_point == CrashPoint::Message, - data_directory: config.data_directory.clone(), - persistent_config_opt: None, + persistent_config_factory: Box::new(PersistentConfigurationFactoryReal::new( + config.data_directory.clone(), + )), + persistent_config: Box::new(PersistentConfigurationInvalid::new()), db_password_opt: config.db_password_opt.clone(), logger: Logger::new("Neighborhood"), tools: NeighborhoodTools::default(), @@ -467,7 +467,11 @@ impl Neighborhood { fn handle_start_message(&mut self) { debug!(self.logger, "Connecting to persistent database"); - self.connect_database(); + self.persistent_config = self.persistent_config_factory.make(); + self.gossip_acceptor = Box::new(GossipAcceptorReal::new( + self.cryptde.dup(), + self.persistent_config.as_ref(), + )); self.validate_or_replace_min_hops_value(); self.send_debut_gossip_to_all_initial_descriptors(); } @@ -496,9 +500,9 @@ impl Neighborhood { } fn handle_route_query_message(&mut self, msg: RouteQueryMessage) -> Option { - let debug_msg_opt = self.logger.debug_enabled().then(|| format!("{:?}", msg)); + let debug_msg_opt = self.logger.debug_enabled().then(|| format!("{}", msg.host)); let route_result = if self.mode == NeighborhoodModeLight::ZeroHop { - Ok(self.zero_hop_route_response()) + Ok(self.zero_hop_route_response(msg.host)) } else { self.make_round_trip_route(msg) }; @@ -519,19 +523,6 @@ impl Neighborhood { } } - fn connect_database(&mut self) { - if self.persistent_config_opt.is_none() { - let db_initializer = DbInitializerReal::default(); - let conn = db_initializer - .initialize( - &self.data_directory, - DbInitializationConfig::panic_on_migration(), - ) - .unwrap_or_else(|err| db_connection_launch_panic(err, &self.data_directory)); - self.persistent_config_opt = Some(Box::new(PersistentConfigurationReal::from(conn))); - } - } - fn handle_config_change_msg(&mut self, msg: ConfigChangeMsg) { match msg.change { ConfigChange::UpdateWallets(wallet_pair) => { @@ -578,22 +569,21 @@ impl Neighborhood { } fn validate_or_replace_min_hops_value(&mut self) { - if let Some(persistent_config) = self.persistent_config_opt.as_ref() { - let value_in_db = persistent_config - .min_hops() - .expect("Min Hops value is not initialized inside Database"); - let value_in_neighborhood = self.min_hops; - if value_in_neighborhood != value_in_db { - info!( - self.logger, - "Database with different min hops value detected; \ - currently set: {:?}, found in db: {:?}; changing to {:?}", - value_in_neighborhood, - value_in_db, - value_in_db - ); - self.min_hops = value_in_db; - } + let value_in_db = self + .persistent_config + .min_hops() + .expect("Min Hops value is not initialized inside Database"); + let value_in_neighborhood = self.min_hops; + if value_in_neighborhood != value_in_db { + info!( + self.logger, + "Database with different min hops value detected; \ + currently set: {:?}, found in db: {:?}; changing to {:?}", + value_in_neighborhood, + value_in_db, + value_in_db + ); + self.min_hops = value_in_db; } } @@ -732,12 +722,12 @@ impl Neighborhood { }; } - fn to_node_descriptors(&self, keys: &[PublicKey]) -> Vec { + fn to_node_descriptors(&self, keys: &[&PublicKey]) -> Vec { keys.iter() .map(|k| { NodeDescriptor::from(( self.neighborhood_database - .node_by_key(k) + .node_by_key(*k) .expectv("NodeRecord"), self.chain, self.cryptde.as_ref(), @@ -781,37 +771,40 @@ impl Neighborhood { db_patch_size: self.db_patch_size, user_exit_preferences_opt: Some(self.user_exit_preferences.clone()), }; - let acceptance_result = self.gossip_acceptor.handle( + let acceptance_results = self.gossip_acceptor.handle( &mut self.neighborhood_database, agrs, gossip_source, neighborhood_metadata, ); - match acceptance_result { - GossipAcceptanceResult::Accepted => { - self.user_exit_preferences.db_countries = self.init_db_countries(); - self.gossip_to_neighbors() - } - GossipAcceptanceResult::Reply(next_debut, target_key, target_node_addr) => { - //TODO also ensure init_db_countries on hop change - if self.min_hops == Hops::OneHop { - self.user_exit_preferences.db_countries = self.init_db_countries(); + if acceptance_results.is_empty() { + trace!(self.logger, "Gossip from {} ignored", gossip_source); + self.handle_gossip_ignored(&ignored_node_name, gossip_record_count) + } else { + acceptance_results.into_iter().for_each(|acceptance_result| { + match acceptance_result { + GossipAcceptanceResult::Accepted => { + self.user_exit_preferences.db_countries = self.init_db_countries(); + self.gossip_to_neighbors() + } + GossipAcceptanceResult::Reply(next_debut, target_key, target_node_addr) => { + //TODO also ensure init_db_countries on hop change + if self.min_hops == Hops::OneHop { + self.user_exit_preferences.db_countries = self.init_db_countries(); + } + self.handle_gossip_reply(next_debut, &target_key, &target_node_addr) + } + GossipAcceptanceResult::Failed(failure, target_key, target_node_addr) => { + self.handle_gossip_failed(failure, &target_key, &target_node_addr) + } + GossipAcceptanceResult::Ban(reason) => { + // TODO in case we introduce Ban machinery we want to reinitialize the db_countries here as well + // That implies new process in init_db_countries to exclude banned node from the result + warning!(self.logger, "Malefactor detected at {}, but malefactor bans not yet implemented; ignoring: {}", gossip_source, reason); + self.handle_gossip_ignored(&ignored_node_name, gossip_record_count); + } } - self.handle_gossip_reply(next_debut, &target_key, &target_node_addr) - } - GossipAcceptanceResult::Failed(failure, target_key, target_node_addr) => { - self.handle_gossip_failed(failure, &target_key, &target_node_addr) - } - GossipAcceptanceResult::Ignored => { - trace!(self.logger, "Gossip from {} ignored", gossip_source); - self.handle_gossip_ignored(ignored_node_name, gossip_record_count) - } - GossipAcceptanceResult::Ban(reason) => { - // TODO in case we introduce Ban machinery we want to reinitialize the db_countries here as well - // That implies new process in init_db_countries to exclude banned node from the result - warning!(self.logger, "Malefactor detected at {}, but malefactor bans not yet implemented; ignoring: {}", gossip_source, reason); - self.handle_gossip_ignored(ignored_node_name, gossip_record_count); - } + }); } } @@ -820,49 +813,46 @@ impl Neighborhood { neighbor_keys_before: HashSet, neighbor_keys_after: HashSet, ) { - self.curate_past_neighbors(neighbor_keys_before, neighbor_keys_after); + if neighbor_keys_after != neighbor_keys_before { + self.curate_past_neighbors(&neighbor_keys_after); + } else { + debug!( + self.logger, + "No neighbor changes; leaving past_neighbors alone" + ) + } self.check_connectedness(); } - fn curate_past_neighbors( - &mut self, - neighbor_keys_before: HashSet, - neighbor_keys_after: HashSet, - ) { - if neighbor_keys_after != neighbor_keys_before { - if let Some(db_password) = &self.db_password_opt { - let nds = self - .to_node_descriptors(neighbor_keys_after.into_iter().collect_vec().as_slice()); - let node_descriptors_opt = if nds.is_empty() { None } else { Some(nds) }; - debug!( - self.logger, - "Saving neighbor list: {:?}", node_descriptors_opt - ); - match self - .persistent_config_opt - .as_mut() - .expect("PersistentConfig was not set by StartMessage") - .set_past_neighbors(node_descriptors_opt, db_password) - { - Ok(_) => info!(self.logger, "Persisted neighbor changes for next run"), - Err(PersistentConfigError::DatabaseError(msg)) - if &msg == "database is locked" => - { - warning!( + fn curate_past_neighbors(&mut self, neighbor_keys: &HashSet) { + if let Some(db_password) = &self.db_password_opt { + let nds = self.to_node_descriptors(neighbor_keys.iter().collect_vec().as_slice()); + let node_descriptors_opt = if nds.is_empty() { None } else { Some(nds) }; + debug!( + self.logger, + "Saving neighbor list: {:?}", node_descriptors_opt + ); + match self + .persistent_config + .set_past_neighbors(node_descriptors_opt, db_password) + { + Ok(_) => info!(self.logger, "Persisted neighbor changes for next run"), + Err(PersistentConfigError::DatabaseError(msg)) if &msg == "database is locked" => { + warning!( self.logger, "Could not persist immediate-neighbor changes: database locked - skipping" ) - } - Err(e) => error!( - self.logger, - "Could not persist immediate-neighbor changes: {:?}", e - ), - }; - } else { - info!(self.logger, "Declining to persist neighbor changes for next run: no database password supplied") - } + } + Err(e) => error!( + self.logger, + "Could not persist immediate-neighbor changes: {:?}", e + ), + }; } else { - debug!(self.logger, "No neighbor changes; database is unchanged") + info!( + self.logger, + "Declining to persist neighbor changes for next run: no database password supplied" + ) } } @@ -882,7 +872,7 @@ impl Neighborhood { target_component: Component::ProxyClient, return_component_opt: Some(Component::ProxyServer), payload_size: 10000, - hostname_opt: None, + host: Host::new("booga.com", 1234), }; if self.handle_route_query_message(msg).is_some() { debug!( @@ -982,8 +972,7 @@ impl Neighborhood { .expect("route creation error") } - fn zero_hop_route_response(&mut self) -> RouteQueryResponse { - let return_route_id = self.advance_return_route_id(); + fn zero_hop_route_response(&mut self, host: Host) -> RouteQueryResponse { let route = Route::round_trip( RouteSegment::new( vec![self.cryptde.public_key(), self.cryptde.public_key()], @@ -995,17 +984,13 @@ impl Neighborhood { ), self.cryptde.as_ref(), None, - return_route_id, None, ) .expect("Couldn't create route"); RouteQueryResponse { route, - expected_services: ExpectedServices::RoundTrip( - vec![ExpectedService::Nothing, ExpectedService::Nothing], - vec![ExpectedService::Nothing, ExpectedService::Nothing], - return_route_id, - ), + expected_services: RoundTrip(vec![Nothing, Nothing], vec![Nothing, Nothing]), + host, } } @@ -1013,7 +998,7 @@ impl Neighborhood { &mut self, request_msg: RouteQueryMessage, ) -> Result { - let hostname_opt = request_msg.hostname_opt.as_deref(); + let host = request_msg.host; let over = self.make_route_segment( self.cryptde.public_key(), request_msg.target_key_opt.as_ref(), @@ -1021,7 +1006,7 @@ impl Neighborhood { request_msg.target_component, request_msg.payload_size, RouteDirection::Over, - hostname_opt, + &host.name, )?; debug!(self.logger, "Route over: {:?}", over); // Estimate for routing-undesirability calculations. @@ -1038,16 +1023,17 @@ impl Neighborhood { .expect("No return component"), anticipated_response_payload_len, RouteDirection::Back, - hostname_opt, + &host.name, )?; debug!(self.logger, "Route back: {:?}", back); - self.compose_route_query_response(over, back) + self.compose_route_query_response(over, back, host) } fn compose_route_query_response( &mut self, over: RouteSegment, back: RouteSegment, + host: Host, ) -> Result { let segments = vec![&over, &back]; @@ -1070,22 +1056,17 @@ impl Neighborhood { Err(e) => return Err(e), }; - let return_route_id = self.advance_return_route_id(); Ok(RouteQueryResponse { route: Route::round_trip( over, back, self.cryptde.as_ref(), self.consuming_wallet_opt.clone(), - return_route_id, Some(self.chain.rec().contract), ) .expect("Internal error: bad route"), - expected_services: ExpectedServices::RoundTrip( - expected_request_services, - expected_response_services, - return_route_id, - ), + expected_services: RoundTrip(expected_request_services, expected_response_services), + host, }) } @@ -1098,7 +1079,7 @@ impl Neighborhood { target_component: Component, payload_size: usize, direction: RouteDirection, - hostname_opt: Option<&str>, + hostname: &str, ) -> Result { let route_opt = self.find_best_route_segment( origin, @@ -1106,7 +1087,7 @@ impl Neighborhood { minimum_hop_count, payload_size, direction, - hostname_opt, + hostname, ); match route_opt { None => { @@ -1145,20 +1126,20 @@ impl Neighborhood { match self.neighborhood_database.node_by_key(route_segment_key) { Some(node) => { if route_segment_key == self.neighborhood_database.root().public_key() { - Ok(ExpectedService::Nothing) + Ok(Nothing) } else { match (originator_key, exit_key) { (Some(originator_key), Some(exit_key)) if route_segment_key == originator_key || route_segment_key == exit_key => { - Ok(ExpectedService::Exit( + Ok(Exit( route_segment_key.clone(), node.earning_wallet(), *node.rate_pack(), )) } - (Some(_), Some(_)) => Ok(ExpectedService::Routing( + (Some(_), Some(_)) => Ok(Routing( route_segment_key.clone(), node.earning_wallet(), *node.rate_pack(), @@ -1276,11 +1257,7 @@ impl Neighborhood { UndesirabilityType::Relay => { node_record.inner.rate_pack.routing_charge(payload_size) as i64 } - UndesirabilityType::ExitRequest(None) => { - node_record.inner.rate_pack.exit_charge(payload_size) as i64 - + node_record.metadata.country_undesirability as i64 - } - UndesirabilityType::ExitRequest(Some(hostname)) => { + UndesirabilityType::ExitRequest(hostname) => { let exit_undesirability = node_record.inner.rate_pack.exit_charge(payload_size) as i64; let country_undesirability = node_record.metadata.country_undesirability as i64; @@ -1326,12 +1303,6 @@ impl Neighborhood { } } - fn advance_return_route_id(&mut self) -> u32 { - let return_route_id = self.next_return_route_id; - self.next_return_route_id = return_route_id.wrapping_add(1); - return_route_id - } - pub fn find_exit_locations<'a>( &'a self, source: &'a PublicKey, @@ -1350,7 +1321,7 @@ impl Neighborhood { PAYLOAD_ZERO_SIZE, RouteDirection::Over, &mut minimum_undesirability, - None, + "booga.com", true, research_exits, ); @@ -1372,7 +1343,7 @@ impl Neighborhood { minimum_hops: usize, payload_size: usize, direction: RouteDirection, - hostname_opt: Option<&str>, + hostname: &str, ) -> Option> { let mut minimum_undesirability = i64::MAX; let initial_undesirability = @@ -1389,7 +1360,7 @@ impl Neighborhood { payload_size, direction, &mut minimum_undesirability, - hostname_opt, + hostname, false, &mut vec![], ) @@ -1413,7 +1384,7 @@ impl Neighborhood { payload_size: usize, direction: RouteDirection, minimum_undesirability: &mut i64, - hostname_opt: Option<&str>, + hostname: &str, research_neighborhood: bool, research_exits: &mut Vec<&'a PublicKey>, ) -> Vec> { @@ -1459,7 +1430,7 @@ impl Neighborhood { payload_size, direction, minimum_undesirability, - hostname_opt, + hostname, research_neighborhood, research_exits, previous_node, @@ -1481,7 +1452,7 @@ impl Neighborhood { payload_size, direction, minimum_undesirability, - hostname_opt, + hostname, research_neighborhood, research_exits, previous_node, @@ -1499,7 +1470,7 @@ impl Neighborhood { payload_size: usize, direction: RouteDirection, minimum_undesirability: &mut i64, - hostname_opt: Option<&str>, + hostname: &str, research_neighborhood: bool, exits_research: &mut Vec<&'a PublicKey>, previous_node: &NodeRecord, @@ -1530,7 +1501,7 @@ impl Neighborhood { new_hops_remaining, payload_size as u64, direction, - hostname_opt, + hostname, ); self.routing_engine( @@ -1541,7 +1512,7 @@ impl Neighborhood { payload_size, direction, minimum_undesirability, - hostname_opt, + hostname, research_neighborhood, exits_research, ) @@ -1598,11 +1569,11 @@ impl Neighborhood { hops_remaining: usize, payload_size: u64, direction: RouteDirection, - hostname_opt: Option<&str>, + hostname: &str, ) -> i64 { let undesirability_type = match (direction, target_opt) { (RouteDirection::Over, None) if hops_remaining == 0 => { - UndesirabilityType::ExitRequest(hostname_opt) + UndesirabilityType::ExitRequest(hostname) } (RouteDirection::Over, _) => UndesirabilityType::Relay, // The exit-and-relay undesirability is initial_undesirability @@ -1950,7 +1921,7 @@ impl Neighborhood { trace!(self.logger, "Sent GossipFailure_0v1: {}", gossip_failure); } - fn handle_gossip_ignored(&self, _ignored_node_name: String, _gossip_record_count: usize) { + fn handle_gossip_ignored(&self, _ignored_node_name: &str, _gossip_record_count: usize) { // Maybe something here eventually for keeping statistics } @@ -2164,7 +2135,7 @@ impl UserExitPreferences { #[derive(PartialEq, Eq, Debug)] enum UndesirabilityType<'hostname> { Relay, - ExitRequest(Option<&'hostname str>), + ExitRequest(&'hostname str), ExitAndRouteResponse, } @@ -2185,10 +2156,20 @@ impl<'a> ComputedRouteSegment<'a> { #[cfg(test)] mod tests { + use super::*; use actix::Recipient; use actix::System; use itertools::Itertools; use lazy_static::lazy_static; + use masq_lib::constants::{DEFAULT_CHAIN, TLS_PORT}; + use masq_lib::messages::{ + CountryGroups, ToMessageBody, UiConnectionChangeBroadcast, UiConnectionStage, + }; + use masq_lib::test_utils::utils::{ensure_node_home_directory_exists, TEST_DEFAULT_CHAIN}; + use masq_lib::ui_gateway::MessageBody; + use masq_lib::ui_gateway::MessagePath::Conversation; + use masq_lib::ui_gateway::MessageTarget; + use masq_lib::utils::running_test; use serde_cbor; use std::any::TypeId; use std::cell::RefCell; @@ -2201,31 +2182,36 @@ mod tests { use std::thread; use std::time::Duration; use std::time::Instant; + use time::{Date, Month, PrimitiveDateTime, Time}; use tokio::prelude::Future; - use masq_lib::constants::{DEFAULT_CHAIN, TLS_PORT}; - use masq_lib::messages::{ - CountryGroups, ToMessageBody, UiConnectionChangeBroadcast, UiConnectionStage, + use crate::accountant::test_utils::bc_from_earning_wallet; + use crate::bootstrapper::CryptDEPair; + use crate::database::db_initializer::{ + DbInitializationConfig, DbInitializer, DbInitializerReal, }; - use masq_lib::test_utils::utils::{ensure_node_home_directory_exists, TEST_DEFAULT_CHAIN}; - use masq_lib::ui_gateway::MessageBody; - use masq_lib::ui_gateway::MessagePath::Conversation; - use masq_lib::ui_gateway::MessageTarget; - use masq_lib::utils::running_test; - use crate::db_config::persistent_configuration::PersistentConfigError; use crate::neighborhood::gossip::Gossip_0v1; use crate::neighborhood::gossip::{GossipBuilder, GossipNodeRecord}; + use crate::neighborhood::malefactor::Malefactor; use crate::neighborhood::node_record::{NodeRecordInner_0v1, NodeRecordInputs}; + use crate::neighborhood::overall_connection_status::ConnectionStageErrors::{ + NoGossipResponseReceived, PassLoopFound, TcpConnectionFailed, + }; + use crate::neighborhood::overall_connection_status::{ + ConnectionProgress, ConnectionStage, OverallConnectionStage, + }; use crate::stream_messages::{NonClandestineAttributes, RemovedStreamType}; use crate::sub_lib::cryptde::{decodex, encodex, CryptData, PlainData}; use crate::sub_lib::cryptde_null::CryptDENull; use crate::sub_lib::dispatcher::Endpoint; use crate::sub_lib::hop::LiveHop; use crate::sub_lib::hopper::MessageType; + use crate::sub_lib::host::Host; + use crate::sub_lib::neighborhood::ExpectedServices::OneWay; use crate::sub_lib::neighborhood::{ - AskAboutDebutGossipMessage, ConfigChange, ConfigChangeMsg, ExpectedServices, - NeighborhoodMode, WalletPair, + AskAboutDebutGossipMessage, ConfigChange, ConfigChangeMsg, NeighborhoodMode, + RatePackLimits, WalletPair, }; use crate::sub_lib::neighborhood::{NeighborhoodConfig, DEFAULT_RATE_PACK}; use crate::sub_lib::neighborhood::{NeighborhoodMetadata, RatePack}; @@ -2233,6 +2219,7 @@ mod tests { use crate::sub_lib::stream_handler_pool::TransmitDataMsg; use crate::sub_lib::versioned_data::VersionedData; use crate::test_utils::assert_contains; + use crate::test_utils::database_utils::PersistentConfigurationFactoryMock; use crate::test_utils::make_meaningless_route; use crate::test_utils::make_paying_wallet; use crate::test_utils::make_wallet; @@ -2248,27 +2235,17 @@ mod tests { use crate::test_utils::recorder::peer_actors_builder; use crate::test_utils::recorder::Recorder; use crate::test_utils::recorder::Recording; + use crate::test_utils::unshared_test_utils::notify_handlers::NotifyLaterHandleMock; use crate::test_utils::unshared_test_utils::{ assert_on_initialization_with_panic_on_migration, make_cpm_recipient, make_node_to_ui_recipient, make_recipient_and_recording_arc, prove_that_crash_request_handler_is_hooked_up, AssertionsMessage, }; use crate::test_utils::vec_to_set; - - use super::*; - use crate::accountant::test_utils::bc_from_earning_wallet; - use crate::bootstrapper::CryptDEPair; - use crate::neighborhood::overall_connection_status::ConnectionStageErrors::{ - NoGossipResponseReceived, PassLoopFound, TcpConnectionFailed, - }; - use crate::neighborhood::overall_connection_status::{ - ConnectionProgress, ConnectionStage, OverallConnectionStage, - }; - use crate::test_utils::unshared_test_utils::notify_handlers::NotifyLaterHandleMock; use masq_lib::test_utils::logging::{init_test_logging, TestLogHandler}; lazy_static! { - static ref CRYPTDE_PAIR: CryptDEPair = CryptDEPair::null(); + static ref N_CRYPTDE_PAIR: CryptDEPair = CryptDEPair::null(); } impl Neighborhood { @@ -2323,7 +2300,7 @@ mod tests { let neighborhood_config = NeighborhoodConfig { mode, min_hops }; let subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( neighborhood_config, make_wallet("earning"), @@ -2423,12 +2400,17 @@ mod tests { #[test] fn introduction_results_in_full_neighborship_in_debutant_db_and_enrich_db_countries_on_one_hop() { - let debut_node = make_global_cryptde_node_record(1111, true, &CRYPTDE_PAIR); - let mut debut_subject = neighborhood_from_nodes(&debut_node, None, &CRYPTDE_PAIR); + let debut_node = make_global_cryptde_node_record(1111, true, &N_CRYPTDE_PAIR); + let mut debut_subject = neighborhood_from_nodes(&debut_node, None, &N_CRYPTDE_PAIR); debut_subject.min_hops = Hops::OneHop; - let persistent_config = - PersistentConfigurationMock::new().set_past_neighbors_result(Ok(())); - debut_subject.persistent_config_opt = Some(Box::new(persistent_config)); + let persistent_config = PersistentConfigurationMock::new() + .set_past_neighbors_result(Ok(())) + .rate_pack_limits_result(Ok(RatePackLimits::test_default())); + debut_subject.persistent_config = Box::new(persistent_config); + debut_subject.gossip_acceptor = Box::new(GossipAcceptorReal::new( + N_CRYPTDE_PAIR.main.dup(), + debut_subject.persistent_config.as_ref(), + )); let debut_root_key = debut_subject.neighborhood_database.root_key().clone(); let introducer_node = make_node_record_cc(3333, true, "AU"); //AU let introducee = make_node_record_cc(2222, true, "FR"); //FR @@ -2465,7 +2447,7 @@ mod tests { expected = "Neighbor masq://eth-ropsten:AQIDBA@1.2.3.4:1234 is not on the mainnet blockchain" )] fn cant_create_mainnet_neighborhood_with_non_mainnet_neighbors() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = N_CRYPTDE_PAIR.main.as_ref(); let earning_wallet = make_wallet("earning"); let mut bc = bc_from_nc_plus( NeighborhoodConfig { @@ -2482,7 +2464,7 @@ mod tests { ); bc.blockchain_bridge_config.chain = DEFAULT_CHAIN; - let _ = Neighborhood::new(CRYPTDE_PAIR.clone(), &bc); + let _ = Neighborhood::new(N_CRYPTDE_PAIR.clone(), &bc); } #[test] @@ -2490,7 +2472,7 @@ mod tests { expected = "Neighbor masq://eth-mainnet:AQIDBA@1.2.3.4:1234 is on the mainnet blockchain" )] fn cant_create_non_mainnet_neighborhood_with_mainnet_neighbors() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = N_CRYPTDE_PAIR.main.as_ref(); let earning_wallet = make_wallet("earning"); let mut bc = bc_from_nc_plus( NeighborhoodConfig { @@ -2507,16 +2489,16 @@ mod tests { ); bc.blockchain_bridge_config.chain = TEST_DEFAULT_CHAIN; - let _ = Neighborhood::new(CRYPTDE_PAIR.clone(), &bc); + let _ = Neighborhood::new(N_CRYPTDE_PAIR.clone(), &bc); } #[test] fn node_with_zero_hop_config_creates_single_node_database() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = N_CRYPTDE_PAIR.main.as_ref(); let earning_wallet = make_wallet("earning"); let subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::ZeroHop, @@ -2537,12 +2519,12 @@ mod tests { #[test] fn node_with_originate_only_config_is_decentralized_with_neighbor_but_not_ip() { - let cryptde: &dyn CryptDE = CRYPTDE_PAIR.main.as_ref(); + let cryptde: &dyn CryptDE = N_CRYPTDE_PAIR.main.as_ref(); let neighbor: NodeRecord = make_node_record(1234, true); let earning_wallet = make_wallet("earning"); let subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::OriginateOnly( @@ -2583,7 +2565,7 @@ mod tests { let system = System::new("node_with_no_neighbor_configs_ignores_bootstrap_neighborhood_now_message"); let mut subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::ZeroHop, @@ -2594,10 +2576,12 @@ mod tests { "node_with_zero_hop_config_ignores_start_message", ), ); - subject.persistent_config_opt = Some(Box::new( - PersistentConfigurationMock::new().min_hops_result(Ok(MIN_HOPS_FOR_TEST)), + subject.persistent_config_factory = Box::new(PersistentConfigurationFactoryMock::new( + PersistentConfigurationMock::new() + .min_hops_result(Ok(MIN_HOPS_FOR_TEST)) + .rate_pack_limits_result(Ok(RatePackLimits::test_default())), )); - subject.data_directory = data_dir; + // subject.data_directory = data_dir; let addr = subject.start(); let sub = addr.clone().recipient::(); let (hopper, _, hopper_recording_arc) = make_recorder(); @@ -2616,7 +2600,7 @@ mod tests { #[test] fn neighborhood_adds_nodes_and_links() { - let cryptde: &dyn CryptDE = CRYPTDE_PAIR.main.as_ref(); + let cryptde: &dyn CryptDE = N_CRYPTDE_PAIR.main.as_ref(); let earning_wallet = make_wallet("earning"); let consuming_wallet = Some(make_paying_wallet(b"consuming")); let one_neighbor_node = make_node_record(3456, true); @@ -2624,7 +2608,7 @@ mod tests { let this_node_addr = NodeAddr::new(&IpAddr::from_str("5.4.3.2").unwrap(), &[5678]); let subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::Standard( @@ -2723,7 +2707,7 @@ mod tests { }; let bootstrap_config = bc_from_nc_plus(neighborhood_config, make_wallet("earning"), None, "test"); - let mut subject = Neighborhood::new(CRYPTDE_PAIR.clone(), &bootstrap_config); + let mut subject = Neighborhood::new(N_CRYPTDE_PAIR.clone(), &bootstrap_config); subject .overall_connection_status .get_connection_progress_by_ip(peer_1) @@ -3176,7 +3160,7 @@ mod tests { } #[test] - pub fn progress_in_the_stage_of_overall_connection_status_made_by_one_cpm_is_not_overriden_by_the_other( + pub fn progress_in_the_stage_of_overall_connection_status_made_by_one_cpm_is_not_overridden_by_the_other( ) { let peer_1 = make_ip(1); let peer_2 = make_ip(2); @@ -3191,12 +3175,12 @@ mod tests { min_hops: MIN_HOPS_FOR_TEST, }; let mut subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( neighborhood_config, make_wallet("earning"), None, - "progress_in_the_stage_of_overall_connection_status_made_by_one_cpm_is_not_overriden_by_the_other"), + "progress_in_the_stage_of_overall_connection_status_made_by_one_cpm_is_not_overridden_by_the_other"), ); let (node_to_ui_recipient, _) = make_node_to_ui_recipient(); subject.node_to_ui_recipient_opt = Some(node_to_ui_recipient); @@ -3243,14 +3227,14 @@ mod tests { #[test] fn gossip_failures_eventually_stop_the_neighborhood() { init_test_logging(); - let cryptde: &dyn CryptDE = CRYPTDE_PAIR.main.as_ref(); + let cryptde: &dyn CryptDE = N_CRYPTDE_PAIR.main.as_ref(); let earning_wallet = make_wallet("earning"); let one_neighbor_node: NodeRecord = make_node_record(3456, true); let another_neighbor_node: NodeRecord = make_node_record(4567, true); let this_node_addr = NodeAddr::new(&IpAddr::from_str("5.4.3.2").unwrap(), &[5678]); let subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::Standard( @@ -3275,14 +3259,14 @@ mod tests { let ecp1 = ExpiredCoresPackage::new( one_neighbor_node.node_addr_opt().unwrap().into(), None, - make_meaningless_route(&CRYPTDE_PAIR), + make_meaningless_route(&N_CRYPTDE_PAIR), GossipFailure_0v1::NoNeighbors, 0, ); let ecp2 = ExpiredCoresPackage::new( another_neighbor_node.node_addr_opt().unwrap().into(), None, - make_meaningless_route(&CRYPTDE_PAIR), + make_meaningless_route(&N_CRYPTDE_PAIR), GossipFailure_0v1::ManualRejection, 0, ); @@ -3309,7 +3293,10 @@ mod tests { let addr: Addr = subject.start(); let sub: Recipient = addr.recipient::(); - let future = sub.send(RouteQueryMessage::data_indefinite_route_request(None, 400)); + let future = sub.send(RouteQueryMessage::data_indefinite_route_request( + Host::new("booga.com", 1234), + 400, + )); System::current().stop_with_code(0); system.run(); @@ -3325,7 +3312,10 @@ mod tests { let addr: Addr = subject.start(); let sub: Recipient = addr.recipient::(); - let future = sub.send(RouteQueryMessage::data_indefinite_route_request(None, 430)); + let future = sub.send(RouteQueryMessage::data_indefinite_route_request( + Host::new("booga.com", 1234), + 430, + )); System::current().stop_with_code(0); system.run(); @@ -3335,7 +3325,7 @@ mod tests { #[test] fn route_query_works_when_node_is_set_for_one_hop_and_no_consuming_wallet() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = N_CRYPTDE_PAIR.main.as_ref(); let earning_wallet = make_wallet("earning"); let system = System::new("route_query_works_when_node_is_set_for_one_hop_and_no_consuming_wallet"); @@ -3365,7 +3355,8 @@ mod tests { } let addr: Addr = subject.start(); let sub: Recipient = addr.recipient::(); - let msg = RouteQueryMessage::data_indefinite_route_request(None, 54000); + let msg = + RouteQueryMessage::data_indefinite_route_request(Host::new("booga.com", 1234), 54000); let future = sub.send(msg); @@ -3390,29 +3381,28 @@ mod tests { ), cryptde, None, - 0, None, ) .unwrap(), - expected_services: ExpectedServices::RoundTrip( + expected_services: RoundTrip( vec![ - ExpectedService::Nothing, - ExpectedService::Exit( + Nothing, + Exit( desirable_exit_node.public_key().clone(), desirable_exit_node.earning_wallet(), rate_pack(2345), ), ], vec![ - ExpectedService::Exit( + Exit( desirable_exit_node.public_key().clone(), desirable_exit_node.earning_wallet(), rate_pack(2345), ), - ExpectedService::Nothing, + Nothing, ], - 0, ), + host: Host::new("booga.com", 1234), }; assert_eq!(expected_response, result); } @@ -3425,7 +3415,8 @@ mod tests { subject.min_hops = Hops::TwoHops; let addr: Addr = subject.start(); let sub: Recipient = addr.recipient::(); - let msg = RouteQueryMessage::data_indefinite_route_request(None, 20000); + let msg = + RouteQueryMessage::data_indefinite_route_request(Host::new("booga.com", 1234), 20000); let future = sub.send(msg); @@ -3437,7 +3428,7 @@ mod tests { #[test] fn route_query_responds_with_standard_zero_hop_route_when_requested() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = N_CRYPTDE_PAIR.main.as_ref(); let system = System::new("responds_with_standard_zero_hop_route_when_requested"); let mut subject = make_standard_subject(); subject.mode = NeighborhoodModeLight::ZeroHop; @@ -3445,7 +3436,8 @@ mod tests { let sub: Recipient = addr.recipient::(); let future = sub.send(RouteQueryMessage::data_indefinite_route_request( - None, 12345, + Host::new("google.com", 1234), + 12345, )); System::current().stop_with_code(0); @@ -3463,39 +3455,15 @@ mod tests { ), cryptde, None, - 0, None, ) .unwrap(), - expected_services: ExpectedServices::RoundTrip( - vec![ExpectedService::Nothing, ExpectedService::Nothing], - vec![ExpectedService::Nothing, ExpectedService::Nothing], - 0, - ), + expected_services: RoundTrip(vec![Nothing, Nothing], vec![Nothing, Nothing]), + host: Host::new("google.com", 1234), }; assert_eq!(result, expected_response); } - #[test] - fn zero_hop_routing_handles_return_route_id_properly() { - let mut subject = make_standard_subject(); - let result0 = subject.zero_hop_route_response(); - let result1 = subject.zero_hop_route_response(); - - let return_route_id_0 = match result0.expected_services { - ExpectedServices::RoundTrip(_, _, id) => id, - _ => panic!("expected RoundTrip got OneWay"), - }; - - let return_route_id_1 = match result1.expected_services { - ExpectedServices::RoundTrip(_, _, id) => id, - _ => panic!("expected RoundTrip got OneWay"), - }; - - assert_eq!(return_route_id_0, 0); - assert_eq!(return_route_id_1, 1); - } - /* Database: @@ -3508,7 +3476,7 @@ mod tests { #[test] fn route_query_messages() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = N_CRYPTDE_PAIR.main.as_ref(); let earning_wallet = make_wallet("earning"); let system = System::new("route_query_messages"); let mut subject = make_standard_subject(); @@ -3541,7 +3509,10 @@ mod tests { let addr: Addr = subject.start(); let sub: Recipient = addr.recipient::(); - let data_route = sub.send(RouteQueryMessage::data_indefinite_route_request(None, 5000)); + let data_route = sub.send(RouteQueryMessage::data_indefinite_route_request( + Host::new("booga.com", 1234), + 5000, + )); System::current().stop_with_code(0); system.run(); @@ -3554,41 +3525,24 @@ mod tests { segment(&[r, q, p], &Component::ProxyServer), cryptde, consuming_wallet_opt, - 0, Some(contract_address), ) .unwrap(), - expected_services: ExpectedServices::RoundTrip( + expected_services: RoundTrip( vec![ - ExpectedService::Nothing, - ExpectedService::Routing( - q.public_key().clone(), - q.earning_wallet(), - rate_pack(3456), - ), - ExpectedService::Exit( - r.public_key().clone(), - r.earning_wallet(), - rate_pack(4567), - ), + Nothing, + Routing(q.public_key().clone(), q.earning_wallet(), rate_pack(3456)), + Exit(r.public_key().clone(), r.earning_wallet(), rate_pack(4567)), ], vec![ - ExpectedService::Exit( - r.public_key().clone(), - r.earning_wallet(), - rate_pack(4567), - ), - ExpectedService::Routing( - q.public_key().clone(), - q.earning_wallet(), - rate_pack(3456), - ), - ExpectedService::Nothing, + Exit(r.public_key().clone(), r.earning_wallet(), rate_pack(4567)), + Routing(q.public_key().clone(), q.earning_wallet(), rate_pack(3456)), + Nothing, ], - 0, ), + host: Host::new("booga.com", 1234), }; - assert_eq!(expected_response, result); + assert_eq!(result, expected_response); } #[test] @@ -3598,6 +3552,7 @@ mod tests { let result: Result = subject.compose_route_query_response( RouteSegment::new(vec![], Component::Neighborhood), RouteSegment::new(vec![], Component::Neighborhood), + Host::new("booga.com", 1234), ); assert!(result.is_err()); let error_expectation: String = result.expect_err("Expected an Err but got:"); @@ -3607,18 +3562,6 @@ mod tests { ); } - #[test] - fn next_return_route_id_wraps_around() { - let mut subject = make_standard_subject(); - subject.next_return_route_id = 0xFFFFFFFF; - - let end = subject.advance_return_route_id(); - let beginning = subject.advance_return_route_id(); - - assert_eq!(end, 0xFFFFFFFF); - assert_eq!(beginning, 0x00000000); - } - /* Database: @@ -3627,37 +3570,6 @@ mod tests { Tests will be written from the viewpoint of O. */ - #[test] - fn return_route_ids_increase() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); - let system = System::new("return_route_ids_increase"); - let (_, _, _, mut subject) = make_o_r_e_subject(); - subject.min_hops = Hops::TwoHops; - let addr: Addr = subject.start(); - let sub: Recipient = addr.recipient::(); - - let data_route_0 = sub.send(RouteQueryMessage::data_indefinite_route_request(None, 2000)); - let data_route_1 = sub.send(RouteQueryMessage::data_indefinite_route_request(None, 3000)); - - System::current().stop_with_code(0); - system.run(); - let result_0 = data_route_0.wait().unwrap().unwrap(); - let result_1 = data_route_1.wait().unwrap().unwrap(); - let juicy_parts = |result: RouteQueryResponse| { - let last_element = result.route.hops.last().unwrap(); - let last_element_dec = cryptde.decode(last_element).unwrap(); - let network_return_route_id: u32 = - serde_cbor::de::from_slice(last_element_dec.as_slice()).unwrap(); - let metadata_return_route_id = match result.expected_services { - ExpectedServices::RoundTrip(_, _, id) => id, - _ => panic!("expected RoundTrip got OneWay"), - }; - (network_return_route_id, metadata_return_route_id) - }; - assert_eq!(juicy_parts(result_0), (0, 0)); - assert_eq!(juicy_parts(result_1), (1, 1)); - } - #[test] fn handle_neighborhood_graph_message_works() { let test_name = "handle_neighborhood_graph_message_works"; @@ -4518,6 +4430,7 @@ mod tests { let result: Result = subject.compose_route_query_response( RouteSegment::new(vec![], Component::ProxyClient), RouteSegment::new(vec![], Component::ProxyServer), + Host::new("booga.com", 1234), ); assert!(result.is_err()); let error_expectation: String = result.expect_err("Expected an Err but got:"); @@ -4534,6 +4447,7 @@ mod tests { let result: Result = subject.compose_route_query_response( RouteSegment::new(vec![&PublicKey::new(&[3, 3, 8])], Component::ProxyClient), RouteSegment::new(vec![&PublicKey::new(&[8, 3, 3])], Component::ProxyServer), + Host::new("booga.com", 1234), ); assert!(result.is_err()); let error_expectation: String = result.expect_err("Expected an Err but got:"); @@ -4541,7 +4455,6 @@ mod tests { error_expectation, "Cannot make multi_hop with unknown neighbor" ); - assert_eq!(subject.next_return_route_id, 0); } #[test] @@ -4663,34 +4576,58 @@ mod tests { // At least two hops from p to anywhere standard let route_opt = - subject.find_best_route_segment(p, None, 2, 10000, RouteDirection::Over, None); + subject.find_best_route_segment(p, None, 2, 10000, RouteDirection::Over, "booga.com"); assert_eq!(route_opt.unwrap(), vec![p, s, t]); // no [p, r, s] or [p, s, r] because s and r are both neighbors of p and can't exit for it // At least two hops over from p to t - let route_opt = - subject.find_best_route_segment(p, Some(t), 2, 10000, RouteDirection::Over, None); + let route_opt = subject.find_best_route_segment( + p, + Some(t), + 2, + 10000, + RouteDirection::Over, + "booga.com", + ); assert_eq!(route_opt.unwrap(), vec![p, s, t]); // At least two hops over from t to p - let route_opt = - subject.find_best_route_segment(t, Some(p), 2, 10000, RouteDirection::Over, None); + let route_opt = subject.find_best_route_segment( + t, + Some(p), + 2, + 10000, + RouteDirection::Over, + "booga.com", + ); assert_eq!(route_opt, None); // p is consume-only; can't be an exit Node. // At least two hops back from t to p - let route_opt = - subject.find_best_route_segment(t, Some(p), 2, 10000, RouteDirection::Back, None); + let route_opt = subject.find_best_route_segment( + t, + Some(p), + 2, + 10000, + RouteDirection::Back, + "booga.com", + ); assert_eq!(route_opt.unwrap(), vec![t, s, p]); // p is consume-only, but it's the originating Node, so including it is okay // At least two hops from p to Q - impossible - let route_opt = - subject.find_best_route_segment(p, Some(q), 2, 10000, RouteDirection::Over, None); + let route_opt = subject.find_best_route_segment( + p, + Some(q), + 2, + 10000, + RouteDirection::Over, + "booga.com", + ); assert_eq!(route_opt, None); } @@ -4735,7 +4672,7 @@ mod tests { 3, 10000, RouteDirection::Back, - None, + "booga.com", ) .unwrap(); @@ -4810,7 +4747,7 @@ mod tests { 3, 10000, RouteDirection::Over, - None, + "booga.com", ); let after = Instant::now(); @@ -4861,8 +4798,14 @@ mod tests { db.add_arbitrary_full_neighbor(c_au_key, a_fr_key); subject.handle_exit_location_message(message, 0, 0); - let route_cz = - subject.find_best_route_segment(root_key, None, 2, 10000, RouteDirection::Over, None); + let route_cz = subject.find_best_route_segment( + root_key, + None, + 2, + 10000, + RouteDirection::Over, + "booga.com", + ); assert_eq!(route_cz, None); } @@ -4927,7 +4870,7 @@ mod tests { subject_min_hops, 10000, RouteDirection::Over, - None, + "booga.com", ); let exit_node = cdb.node_by_key(&route_au.as_ref().unwrap().last().unwrap()); @@ -4982,8 +4925,14 @@ mod tests { }; subject.handle_exit_location_message(message, 0, 0); - let route_fr = - subject.find_best_route_segment(root_key, None, 2, 10000, RouteDirection::Over, None); + let route_fr = subject.find_best_route_segment( + root_key, + None, + 2, + 10000, + RouteDirection::Over, + "booga.com", + ); let exit_node = cdb.node_by_key(&route_fr.as_ref().unwrap().last().unwrap()); assert_eq!( @@ -5006,7 +4955,7 @@ mod tests { // At least two hops from P to anywhere standard let route_opt = - subject.find_best_route_segment(p, None, 2, 10000, RouteDirection::Over, None); + subject.find_best_route_segment(p, None, 2, 10000, RouteDirection::Over, "booga.com"); assert_eq!(route_opt, None); } @@ -5023,7 +4972,7 @@ mod tests { 5, // Lots of hops to go yet 1_000, RouteDirection::Over, - Some("hostname.com"), + "hostname.com", ); let rate_pack = node_record.rate_pack(); @@ -5047,7 +4996,7 @@ mod tests { 0, // Last hop 1_000, RouteDirection::Over, - Some("hostname.com"), + "hostname.com", ); let rate_pack = node_record.rate_pack(); @@ -5075,7 +5024,7 @@ mod tests { 0, // Last hop 1_000, RouteDirection::Over, - Some("hostname.com"), + "hostname.com", ); let rate_pack = node_record.rate_pack(); @@ -5148,7 +5097,7 @@ mod tests { 5, // Plenty of hops remaining: not there yet 1_000, RouteDirection::Back, - None, + "booga.com", ); let rate_pack = node_record.rate_pack(); @@ -5162,7 +5111,7 @@ mod tests { #[test] fn gossips_after_removing_a_neighbor() { let (hopper, hopper_awaiter, hopper_recording) = make_recorder(); - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = N_CRYPTDE_PAIR.main.as_ref(); let earning_wallet = make_wallet("earning"); let consuming_wallet = Some(make_paying_wallet(b"consuming")); let this_node = NodeRecord::new_for_tests( @@ -5185,7 +5134,7 @@ mod tests { thread::spawn(move || { let system = System::new("gossips_after_removing_a_neighbor"); let mut subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::Standard( @@ -5301,10 +5250,10 @@ mod tests { let handle_params_arc = Arc::new(Mutex::new(vec![])); let gossip_acceptor = GossipAcceptorMock::new() .handle_params(&handle_params_arc) - .handle_result(GossipAcceptanceResult::Ignored); - let mut subject_node = make_global_cryptde_node_record(1234, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + .handle_result(vec![]); + let mut subject_node = make_global_cryptde_node_record(1234, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let neighbor = make_node_record(1111, true); - let mut subject = neighborhood_from_nodes(&subject_node, Some(&neighbor), &CRYPTDE_PAIR); + let mut subject = neighborhood_from_nodes(&subject_node, Some(&neighbor), &N_CRYPTDE_PAIR); subject.gossip_acceptor = Box::new(gossip_acceptor); let gossip = GossipBuilder::new(&subject.neighborhood_database) .node(subject_node.public_key(), true) @@ -5312,7 +5261,7 @@ mod tests { let cores_package = ExpiredCoresPackage { immediate_neighbor: subject_node.node_addr_opt().unwrap().into(), paying_wallet: None, - remaining_route: make_meaningless_route(&CRYPTDE_PAIR), + remaining_route: make_meaningless_route(&N_CRYPTDE_PAIR), payload: gossip.clone(), payload_len: 0, }; @@ -5345,9 +5294,9 @@ mod tests { #[test] fn neighborhood_sends_only_an_acceptance_debut_when_an_acceptance_debut_is_provided() { let introduction_target_node = make_node_record(7345, true); - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let neighbor = make_node_record(1050, true); - let mut subject = neighborhood_from_nodes(&subject_node, Some(&neighbor), &CRYPTDE_PAIR); + let mut subject = neighborhood_from_nodes(&subject_node, Some(&neighbor), &N_CRYPTDE_PAIR); subject .neighborhood_database .add_node(introduction_target_node.clone()) @@ -5361,11 +5310,11 @@ mod tests { .node(subject_node.public_key(), true) .build(); let gossip_acceptor = - GossipAcceptorMock::new().handle_result(GossipAcceptanceResult::Reply( + GossipAcceptorMock::new().handle_result(vec![GossipAcceptanceResult::Reply( debut.clone(), introduction_target_node.public_key().clone(), introduction_target_node.node_addr_opt().unwrap(), - )); + )]); subject.gossip_acceptor = Box::new(gossip_acceptor); let (hopper, _, hopper_recording_arc) = make_recorder(); let peer_actors = peer_actors_builder().hopper(hopper).build(); @@ -5396,18 +5345,18 @@ mod tests { #[test] fn neighborhood_transmits_gossip_failure_properly() { - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let neighbor = make_node_record(1111, true); let public_key = PublicKey::new(&[1, 2, 3, 4]); let node_addr = NodeAddr::from_str("1.2.3.4:1234").unwrap(); let gossip_acceptor = - GossipAcceptorMock::new().handle_result(GossipAcceptanceResult::Failed( + GossipAcceptorMock::new().handle_result(vec![GossipAcceptanceResult::Failed( GossipFailure_0v1::NoSuitableNeighbors, public_key.clone(), node_addr.clone(), - )); + )]); let mut subject: Neighborhood = - neighborhood_from_nodes(&subject_node, Some(&neighbor), &CRYPTDE_PAIR); + neighborhood_from_nodes(&subject_node, Some(&neighbor), &N_CRYPTDE_PAIR); let (hopper, _, hopper_recording_arc) = make_recorder(); let system = System::new("neighborhood_transmits_gossip_failure_properly"); let peer_actors = peer_actors_builder().hopper(hopper).build(); @@ -5451,7 +5400,7 @@ mod tests { _agrs: Vec, _gossip_source: SocketAddr, _neighborhood_metadata: NeighborhoodMetadata, - ) -> GossipAcceptanceResult { + ) -> Vec { let non_root_database_keys = database .keys() .into_iter() @@ -5485,7 +5434,7 @@ mod tests { database.add_arbitrary_half_neighbor(k, n); }); }); - GossipAcceptanceResult::Ignored + vec![] } } @@ -5497,10 +5446,10 @@ mod tests { #[test] fn neighborhood_does_not_start_accountant_if_no_route_can_be_made() { - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let neighbor = make_node_record(1111, true); let mut subject: Neighborhood = - neighborhood_from_nodes(&subject_node, Some(&neighbor), &CRYPTDE_PAIR); + neighborhood_from_nodes(&subject_node, Some(&neighbor), &N_CRYPTDE_PAIR); let mut replacement_database = subject.neighborhood_database.clone(); replacement_database.add_node(neighbor.clone()).unwrap(); replacement_database @@ -5528,10 +5477,10 @@ mod tests { #[test] fn neighborhood_does_not_start_accountant_if_already_connected() { - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let neighbor = make_node_record(1111, true); let mut subject: Neighborhood = - neighborhood_from_nodes(&subject_node, Some(&neighbor), &CRYPTDE_PAIR); + neighborhood_from_nodes(&subject_node, Some(&neighbor), &N_CRYPTDE_PAIR); let replacement_database = subject.neighborhood_database.clone(); subject.gossip_acceptor = Box::new(DatabaseReplacementGossipAcceptor { replacement_database, @@ -5559,14 +5508,17 @@ mod tests { let (ui_gateway, _, _) = make_recorder(); let mut subject = make_neighborhood_with_linearly_connected_nodes(4); subject.node_to_ui_recipient_opt = Some(ui_gateway.start().recipient()); + subject.db_password_opt = None; let peer_actors = peer_actors_builder().accountant(accountant).build(); bind_subject(&mut subject, peer_actors); let system = System::new("neighborhood_does_not_start_accountant_if_no_route_can_be_made"); - subject.handle_gossip_agrs( - vec![], - SocketAddr::from_str("1.2.3.4:1234").unwrap(), - make_cpm_recipient().0, + subject.handle_database_changes( + // Just different HashSets; the values don't mean anything + vec![subject.neighborhood_database.root().public_key().clone()] + .into_iter() + .collect(), + vec![].into_iter().collect(), ); System::current().stop(); @@ -5582,12 +5534,12 @@ mod tests { let handle_params_arc = Arc::new(Mutex::new(vec![])); let gossip_acceptor = GossipAcceptorMock::new() .handle_params(&handle_params_arc) - .handle_result(GossipAcceptanceResult::Ignored); + .handle_result(vec![]); let (node_to_ui_recipient, _) = make_node_to_ui_recipient(); let peer_1 = make_node_record(1234, true); let peer_2 = make_node_record(6721, true); - let desc_1 = peer_1.node_descriptor(Chain::Dev, CRYPTDE_PAIR.main.as_ref()); - let desc_2 = peer_2.node_descriptor(Chain::Dev, CRYPTDE_PAIR.main.as_ref()); + let desc_1 = peer_1.node_descriptor(Chain::Dev, N_CRYPTDE_PAIR.main.as_ref()); + let desc_2 = peer_2.node_descriptor(Chain::Dev, N_CRYPTDE_PAIR.main.as_ref()); let this_node = make_node_record(7777, true); let initial_node_descriptors = vec![desc_1, desc_2]; let neighborhood_config = NeighborhoodConfig { @@ -5600,7 +5552,7 @@ mod tests { }; let bootstrap_config = bc_from_nc_plus(neighborhood_config, make_wallet("earning"), None, "test"); - let mut subject = Neighborhood::new(CRYPTDE_PAIR.clone(), &bootstrap_config); + let mut subject = Neighborhood::new(N_CRYPTDE_PAIR.clone(), &bootstrap_config); subject.node_to_ui_recipient_opt = Some(node_to_ui_recipient); subject.gossip_acceptor = Box::new(gossip_acceptor); subject.db_patch_size = 6; @@ -5622,6 +5574,14 @@ mod tests { } fn assert_connectivity_check(hops: Hops) { + assert_connectivity_check_with_neighbor_changes_specified(hops, false); + assert_connectivity_check_with_neighbor_changes_specified(hops, true); + } + + fn assert_connectivity_check_with_neighbor_changes_specified( + hops: Hops, + neighbor_changes: bool, + ) { init_test_logging(); let test_name = &format!("connectivity_check_for_{}_hops", hops as usize); let nodes_count = hops as u16 + 1; @@ -5635,13 +5595,18 @@ mod tests { subject.logger = Logger::new(test_name); subject.node_to_ui_recipient_opt = Some(node_to_ui_recipient); subject.connected_signal_opt = Some(connected_signal); + subject.db_password_opt = None; + let neighbor_keys_before: HashSet = match neighbor_changes { + // Just make neighbors before different from neighbors after; the actual keys don't matter + true => vec![subject.neighborhood_database.root().public_key().clone()] + .into_iter() + .collect(), + // Just make neighbors before the same as neighbors after; the actual keys don't matter + false => HashSet::new(), + }; let system = System::new(test_name); - subject.handle_gossip_agrs( - vec![], - SocketAddr::from_str("1.2.3.4:1234").unwrap(), - make_cpm_recipient().0, - ); + subject.handle_database_changes(neighbor_keys_before, vec![].into_iter().collect()); System::current().stop(); system.run(); @@ -5692,12 +5657,15 @@ mod tests { subject.node_to_ui_recipient_opt = Some(node_to_ui_recipient); subject.connected_signal_opt = Some(connected_signal); subject.min_hops = Hops::FiveHops; + subject.db_password_opt = None; let system = System::new(test_name); - subject.handle_gossip_agrs( - vec![], - SocketAddr::from_str("1.2.3.4:1234").unwrap(), - make_cpm_recipient().0, + subject.handle_database_changes( + // Just different HashSets; the values don't mean anything + vec![subject.neighborhood_database.root().public_key().clone()] + .into_iter() + .collect(), + vec![].into_iter().collect(), ); System::current().stop(); @@ -5725,7 +5693,7 @@ mod tests { _agrs: Vec, _gossip_source: SocketAddr, _neighborhood_metadata: NeighborhoodMetadata, - ) -> GossipAcceptanceResult { + ) -> Vec { let half_neighbor_keys = database .root() .half_neighbor_keys() @@ -5740,18 +5708,18 @@ mod tests { database.add_node(nr.clone()).unwrap(); database.add_arbitrary_full_neighbor(&root_key, nr.public_key()); }); - GossipAcceptanceResult::Ignored + vec![] } } #[test] fn neighborhood_updates_past_neighbors_when_neighbor_list_changes() { - let cryptde: &dyn CryptDE = CRYPTDE_PAIR.main.as_ref(); - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let cryptde: &dyn CryptDE = N_CRYPTDE_PAIR.main.as_ref(); + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let old_neighbor = make_node_record(1111, true); let new_neighbor = make_node_record(2222, true); let mut subject: Neighborhood = - neighborhood_from_nodes(&subject_node, Some(&old_neighbor), &CRYPTDE_PAIR); + neighborhood_from_nodes(&subject_node, Some(&old_neighbor), &N_CRYPTDE_PAIR); subject .neighborhood_database .add_node(old_neighbor.clone()) @@ -5767,7 +5735,7 @@ mod tests { .set_past_neighbors_params(&set_past_neighbors_params_arc) .set_past_neighbors_result(Ok(())); subject.gossip_acceptor = Box::new(gossip_acceptor); - subject.persistent_config_opt = Some(Box::new(persistent_config)); + subject.persistent_config = Box::new(persistent_config); subject.handle_gossip_agrs( vec![], @@ -5792,10 +5760,10 @@ mod tests { #[test] fn neighborhood_removes_past_neighbors_when_neighbor_list_goes_empty() { - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let neighbor = make_node_record(1111, true); let mut subject: Neighborhood = - neighborhood_from_nodes(&subject_node, Some(&neighbor), &CRYPTDE_PAIR); + neighborhood_from_nodes(&subject_node, Some(&neighbor), &N_CRYPTDE_PAIR); subject .neighborhood_database .add_node(neighbor.clone()) @@ -5811,7 +5779,7 @@ mod tests { .set_past_neighbors_params(&set_past_neighbors_params_arc) .set_past_neighbors_result(Ok(())); subject.gossip_acceptor = Box::new(gossip_acceptor); - subject.persistent_config_opt = Some(Box::new(persistent_config)); + subject.persistent_config = Box::new(persistent_config); subject.handle_gossip_agrs( vec![], @@ -5827,10 +5795,10 @@ mod tests { #[test] fn neighborhood_does_not_update_past_neighbors_when_neighbor_list_does_not_change() { - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let steadfast_neighbor = make_node_record(1111, true); let mut subject: Neighborhood = - neighborhood_from_nodes(&subject_node, Some(&steadfast_neighbor), &CRYPTDE_PAIR); + neighborhood_from_nodes(&subject_node, Some(&steadfast_neighbor), &N_CRYPTDE_PAIR); subject .neighborhood_database .add_node(steadfast_neighbor.clone()) @@ -5846,7 +5814,7 @@ mod tests { let persistent_config = PersistentConfigurationMock::new() .set_past_neighbors_params(&set_past_neighbors_params_arc); subject.gossip_acceptor = Box::new(gossip_acceptor); - subject.persistent_config_opt = Some(Box::new(persistent_config)); + subject.persistent_config = Box::new(persistent_config); subject.handle_gossip_agrs( vec![], @@ -5861,11 +5829,11 @@ mod tests { #[test] fn neighborhood_does_not_update_past_neighbors_without_password_even_when_neighbor_list_changes( ) { - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let old_neighbor = make_node_record(1111, true); let new_neighbor = make_node_record(2222, true); let mut subject: Neighborhood = - neighborhood_from_nodes(&subject_node, Some(&old_neighbor), &CRYPTDE_PAIR); + neighborhood_from_nodes(&subject_node, Some(&old_neighbor), &N_CRYPTDE_PAIR); subject .neighborhood_database .add_node(old_neighbor.clone()) @@ -5880,7 +5848,7 @@ mod tests { let persistent_config = PersistentConfigurationMock::new() .set_past_neighbors_params(&set_past_neighbors_params_arc); subject.gossip_acceptor = Box::new(gossip_acceptor); - subject.persistent_config_opt = Some(Box::new(persistent_config)); + subject.persistent_config = Box::new(persistent_config); subject.db_password_opt = None; subject.handle_gossip_agrs( @@ -5896,11 +5864,11 @@ mod tests { #[test] fn neighborhood_warns_when_past_neighbors_update_fails_because_of_database_lock() { init_test_logging(); - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let old_neighbor = make_node_record(1111, true); let new_neighbor = make_node_record(2222, true); let mut subject: Neighborhood = - neighborhood_from_nodes(&subject_node, Some(&old_neighbor), &CRYPTDE_PAIR); + neighborhood_from_nodes(&subject_node, Some(&old_neighbor), &N_CRYPTDE_PAIR); subject .neighborhood_database .add_node(old_neighbor.clone()) @@ -5915,7 +5883,7 @@ mod tests { PersistentConfigError::DatabaseError("database is locked".to_string()), )); subject.gossip_acceptor = Box::new(gossip_acceptor); - subject.persistent_config_opt = Some(Box::new(persistent_config)); + subject.persistent_config = Box::new(persistent_config); subject.handle_gossip_agrs( vec![], @@ -5929,11 +5897,11 @@ mod tests { #[test] fn neighborhood_logs_error_when_past_neighbors_update_fails_for_another_reason() { init_test_logging(); - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let old_neighbor = make_node_record(1111, true); let new_neighbor = make_node_record(2222, true); let mut subject: Neighborhood = - neighborhood_from_nodes(&subject_node, Some(&old_neighbor), &CRYPTDE_PAIR); + neighborhood_from_nodes(&subject_node, Some(&old_neighbor), &N_CRYPTDE_PAIR); subject .neighborhood_database .add_node(old_neighbor.clone()) @@ -5948,7 +5916,7 @@ mod tests { PersistentConfigError::DatabaseError("Booga".to_string()), )); subject.gossip_acceptor = Box::new(gossip_acceptor); - subject.persistent_config_opt = Some(Box::new(persistent_config)); + subject.persistent_config = Box::new(persistent_config); subject.handle_gossip_agrs( vec![], @@ -5962,10 +5930,10 @@ mod tests { #[test] fn handle_new_public_ip_changes_public_ip_and_country_code_nothing_else() { init_test_logging(); - let subject_node = make_global_cryptde_node_record(1234, true, &CRYPTDE_PAIR); + let subject_node = make_global_cryptde_node_record(1234, true, &N_CRYPTDE_PAIR); let neighbor = make_node_record(1050, true); let mut subject: Neighborhood = - neighborhood_from_nodes(&subject_node, Some(&neighbor), &CRYPTDE_PAIR); + neighborhood_from_nodes(&subject_node, Some(&neighbor), &N_CRYPTDE_PAIR); subject .neighborhood_database .root_mut() @@ -6013,9 +5981,9 @@ mod tests { #[test] fn neighborhood_sends_from_gossip_producer_when_acceptance_introductions_are_not_provided() { init_test_logging(); - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let neighbor = make_node_record(1050, true); - let mut subject = neighborhood_from_nodes(&subject_node, Some(&neighbor), &CRYPTDE_PAIR); + let mut subject = neighborhood_from_nodes(&subject_node, Some(&neighbor), &N_CRYPTDE_PAIR); let full_neighbor = make_node_record(1234, true); let half_neighbor = make_node_record(2345, true); subject @@ -6033,7 +6001,7 @@ mod tests { .neighborhood_database .add_arbitrary_half_neighbor(subject_node.public_key(), half_neighbor.public_key()); let gossip_acceptor = - GossipAcceptorMock::new().handle_result(GossipAcceptanceResult::Accepted); + GossipAcceptorMock::new().handle_result(vec![GossipAcceptanceResult::Accepted]); subject.gossip_acceptor = Box::new(gossip_acceptor); let gossip = Gossip_0v1::new(vec![]); let produce_params_arc = Arc::new(Mutex::new(vec![])); @@ -6065,7 +6033,7 @@ mod tests { ( package .route - .next_hop(CRYPTDE_PAIR.main.as_ref()) + .next_hop(N_CRYPTDE_PAIR.main.as_ref()) .unwrap() .public_key, package.payload, @@ -6077,7 +6045,7 @@ mod tests { ( full_neighbor.public_key().clone(), encodex( - CRYPTDE_PAIR.main.as_ref(), + N_CRYPTDE_PAIR.main.as_ref(), full_neighbor.public_key(), &MessageType::Gossip(gossip.clone().into()), ) @@ -6086,7 +6054,7 @@ mod tests { ( half_neighbor.public_key().clone(), encodex( - CRYPTDE_PAIR.main.as_ref(), + N_CRYPTDE_PAIR.main.as_ref(), half_neighbor.public_key(), &MessageType::Gossip(gossip.into()), ) @@ -6110,19 +6078,19 @@ mod tests { ) .as_str(), ); - let key_as_str = format!("{}", CRYPTDE_PAIR.main.as_ref().public_key()); + let key_as_str = format!("{}", N_CRYPTDE_PAIR.main.as_ref().public_key()); tlh.exists_log_containing(&format!("Sent Gossip: digraph db {{ \"src\" [label=\"Gossip From:\\n{}\\n5.5.5.5\"]; \"dest\" [label=\"Gossip To:\\nAQIDBA\\n1.2.3.4\"]; \"src\" -> \"dest\" [arrowhead=empty]; }}", &key_as_str[..8])); tlh.exists_log_containing(&format!("Sent Gossip: digraph db {{ \"src\" [label=\"Gossip From:\\n{}\\n5.5.5.5\"]; \"dest\" [label=\"Gossip To:\\nAgMEBQ\\n2.3.4.5\"]; \"src\" -> \"dest\" [arrowhead=empty]; }}", &key_as_str[..8])); } #[test] fn neighborhood_sends_no_gossip_when_target_does_not_exist() { - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A - // This is ungossippable not because of any attribute of its own, but because the - // GossipProducerMock is set to return None when ordered to target it. + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + // This is ungossippable not because of any attribute of its own, but because the + // GossipProducerMock is set to return None when ordered to target it. let ungossippable = make_node_record(1050, true); let mut subject = - neighborhood_from_nodes(&subject_node, Some(&ungossippable), &CRYPTDE_PAIR); + neighborhood_from_nodes(&subject_node, Some(&ungossippable), &N_CRYPTDE_PAIR); subject .neighborhood_database .add_node(ungossippable.clone()) @@ -6131,7 +6099,7 @@ mod tests { .neighborhood_database .add_arbitrary_full_neighbor(subject_node.public_key(), ungossippable.public_key()); let gossip_acceptor = - GossipAcceptorMock::new().handle_result(GossipAcceptanceResult::Accepted); + GossipAcceptorMock::new().handle_result(vec![GossipAcceptanceResult::Accepted]); subject.gossip_acceptor = Box::new(gossip_acceptor); let produce_params_arc = Arc::new(Mutex::new(vec![])); let gossip_producer = GossipProducerMock::new() @@ -6159,22 +6127,22 @@ mod tests { #[test] fn neighborhood_sends_only_relay_gossip_when_gossip_acceptor_relays() { - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let mut subject = neighborhood_from_nodes( &subject_node, Some(&make_node_record(1111, true)), - &CRYPTDE_PAIR, + &N_CRYPTDE_PAIR, ); let debut_node = make_node_record(1234, true); let debut_gossip = GossipBuilder::new(&subject.neighborhood_database) .node(subject_node.public_key(), true) .build(); let gossip_acceptor = - GossipAcceptorMock::new().handle_result(GossipAcceptanceResult::Reply( + GossipAcceptorMock::new().handle_result(vec![GossipAcceptanceResult::Reply( debut_gossip.clone(), debut_node.public_key().clone(), debut_node.node_addr_opt().unwrap(), - )); + )]); subject.gossip_acceptor = Box::new(gossip_acceptor); let (hopper, _, hopper_recording_arc) = make_recorder(); let peer_actors = peer_actors_builder().hopper(hopper).build(); @@ -6213,11 +6181,10 @@ mod tests { #[test] fn neighborhood_sends_no_gossip_when_gossip_acceptor_ignores() { - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let neighbor = make_node_record(1111, true); - let mut subject = neighborhood_from_nodes(&subject_node, Some(&neighbor), &CRYPTDE_PAIR); - let gossip_acceptor = - GossipAcceptorMock::new().handle_result(GossipAcceptanceResult::Ignored); + let mut subject = neighborhood_from_nodes(&subject_node, Some(&neighbor), &N_CRYPTDE_PAIR); + let gossip_acceptor = GossipAcceptorMock::new().handle_result(vec![]); subject.gossip_acceptor = Box::new(gossip_acceptor); let subject_node = subject.neighborhood_database.root().clone(); let (hopper, _, hopper_recording_arc) = make_recorder(); @@ -6240,11 +6207,29 @@ mod tests { #[test] fn neighborhood_complains_about_inability_to_ban_when_gossip_acceptor_requests_it() { init_test_logging(); - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let neighbor = make_node_record(1111, true); - let mut subject = neighborhood_from_nodes(&subject_node, Some(&neighbor), &CRYPTDE_PAIR); - let gossip_acceptor = GossipAcceptorMock::new() - .handle_result(GossipAcceptanceResult::Ban("Bad guy".to_string())); + let mut subject = neighborhood_from_nodes(&subject_node, Some(&neighbor), &N_CRYPTDE_PAIR); + let public_key = PublicKey::from(&b"BadGuyPublicKey"[..]); + let ip_address = IpAddr::from_str("1.3.2.4").unwrap(); + let earning_wallet = make_wallet("BadGuyEarningWallet"); + let consuming_wallet = make_wallet("BadGuyConsumingWallet"); + let timestamp = PrimitiveDateTime::new( + Date::from_calendar_date(2024, Month::April, 1).unwrap(), + Time::from_hms(3, 4, 5).unwrap(), + ); + let reason = "Bad guy".to_string(); + let gossip_acceptor = + GossipAcceptorMock::new().handle_result(vec![GossipAcceptanceResult::Ban( + Malefactor { + public_key_opt: Some(public_key.clone()), + ip_address_opt: Some(ip_address), + earning_wallet_opt: Some(earning_wallet.clone()), + consuming_wallet_opt: Some(consuming_wallet.clone()), + timestamp: timestamp.clone(), + reason: reason.clone(), + }, + )]); subject.gossip_acceptor = Box::new(gossip_acceptor); let subject_node = subject.neighborhood_database.root().clone(); let (hopper, _, hopper_recording_arc) = make_recorder(); @@ -6263,7 +6248,7 @@ mod tests { let hopper_recording = hopper_recording_arc.lock().unwrap(); assert_eq!(0, hopper_recording.len()); let tlh = TestLogHandler::new(); - tlh.exists_log_containing("WARN: Neighborhood: Malefactor detected at 5.5.5.5:5555, but malefactor bans not yet implemented; ignoring: Bad guy"); + tlh.exists_log_containing("WARN: Neighborhood: Malefactor detected at 5.5.5.5:5555, but malefactor bans not yet implemented; ignoring: Malefactor QmFkR3V5UHVibGljS2V5 at 1.3.2.4 with earning wallet 0x004261644775794561726e696e6757616c6c6574, consuming wallet 0x426164477579436f6e73756d696e6757616c6c65 detected at 2024-04-01 3:04:05.0: Bad guy"); } #[test] @@ -6315,7 +6300,7 @@ mod tests { #[test] fn neighborhood_logs_received_gossip_in_dot_graph_format() { init_test_logging(); - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = N_CRYPTDE_PAIR.main.as_ref(); let this_node = NodeRecord::new_for_tests( &cryptde.public_key(), Some(&NodeAddr::new( @@ -6349,7 +6334,7 @@ mod tests { let cores_package = ExpiredCoresPackage { immediate_neighbor: SocketAddr::from_str("1.2.3.4:1234").unwrap(), paying_wallet: Some(make_paying_wallet(b"consuming")), - remaining_route: make_meaningless_route(&CRYPTDE_PAIR), + remaining_route: make_meaningless_route(&N_CRYPTDE_PAIR), payload: gossip, payload_len: 0, }; @@ -6358,7 +6343,7 @@ mod tests { thread::spawn(move || { let system = System::new(""); let subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::Standard( @@ -6413,15 +6398,15 @@ mod tests { .initialize(&data_dir, DbInitializationConfig::test_default()) .unwrap(); } - let cryptde: &dyn CryptDE = CRYPTDE_PAIR.main.as_ref(); + let cryptde: &dyn CryptDE = N_CRYPTDE_PAIR.main.as_ref(); let debut_target = NodeDescriptor::try_from(( - CRYPTDE_PAIR.main.as_ref(), // Used to provide default cryptde + N_CRYPTDE_PAIR.main.as_ref(), // Used to provide default cryptde "masq://eth-ropsten:AQIDBA@1.2.3.4:1234", )) .unwrap(); let (hopper, _, hopper_recording) = make_recorder(); let mut subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::Standard( @@ -6436,10 +6421,11 @@ mod tests { "node_gossips_to_neighbors_on_startup", ), ); - subject.persistent_config_opt = Some(Box::new( - PersistentConfigurationMock::new().min_hops_result(Ok(MIN_HOPS_FOR_TEST)), + subject.persistent_config_factory = Box::new(PersistentConfigurationFactoryMock::new( + PersistentConfigurationMock::new() + .min_hops_result(Ok(MIN_HOPS_FOR_TEST)) + .rate_pack_limits_result(Ok(RatePackLimits::test_default())), )); - subject.data_directory = data_dir; subject.logger = Logger::new("node_gossips_to_neighbors_on_startup"); let this_node = subject.neighborhood_database.root().clone(); let system = System::new("node_gossips_to_neighbors_on_startup"); @@ -6478,7 +6464,7 @@ mod tests { let min_hops_in_neighborhood = Hops::SixHops; let min_hops_in_persistent_configuration = min_hops_in_neighborhood; let mut subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::Standard( @@ -6493,9 +6479,10 @@ mod tests { test_name, ), ); - subject.persistent_config_opt = Some(Box::new( + subject.persistent_config_factory = Box::new(PersistentConfigurationFactoryMock::new( PersistentConfigurationMock::new() - .min_hops_result(Ok(min_hops_in_persistent_configuration)), + .min_hops_result(Ok(min_hops_in_persistent_configuration)) + .rate_pack_limits_result(Ok(RatePackLimits::test_default())), )); let system = System::new(test_name); let addr: Addr = subject.start(); @@ -6521,7 +6508,7 @@ mod tests { let min_hops_in_neighborhood = Hops::SixHops; let min_hops_in_db = Hops::TwoHops; let mut subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::Standard( @@ -6537,8 +6524,10 @@ mod tests { ), ); subject.logger = Logger::new(test_name); - subject.persistent_config_opt = Some(Box::new( - PersistentConfigurationMock::new().min_hops_result(Ok(min_hops_in_db)), + subject.persistent_config_factory = Box::new(PersistentConfigurationFactoryMock::new( + PersistentConfigurationMock::new() + .min_hops_result(Ok(min_hops_in_db)) + .rate_pack_limits_result(Ok(RatePackLimits::test_default())), )); let system = System::new(test_name); let addr: Addr = subject.start(); @@ -6622,7 +6611,7 @@ mod tests { target_component: Component::ProxyClient, return_component_opt: None, payload_size: 10000, - hostname_opt: None, + host: Host::new("booga.com", 1234), }; let unsuccessful_three_hop_route = addr.send(three_hop_route_request); let asserted_node_record = a.clone(); @@ -6647,13 +6636,13 @@ mod tests { } fn node_record_to_neighbor_config(node_record_ref: &NodeRecord) -> NodeDescriptor { - let cryptde: &dyn CryptDE = CRYPTDE_PAIR.main.as_ref(); + let cryptde: &dyn CryptDE = N_CRYPTDE_PAIR.main.as_ref(); NodeDescriptor::from((node_record_ref, Chain::EthRopsten, cryptde)) } #[test] fn neighborhood_sends_node_query_response_with_none_when_initially_configured_with_no_data() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = N_CRYPTDE_PAIR.main.as_ref(); let (recorder, awaiter, recording_arc) = make_recorder(); thread::spawn(move || { let system = System::new("responds_with_none_when_initially_configured_with_no_data"); @@ -6672,7 +6661,7 @@ mod tests { context: TransmitDataMsg { endpoint: Endpoint::Key(cryptde.public_key().clone()), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: Vec::new(), }, recipient, @@ -6692,7 +6681,7 @@ mod tests { #[test] fn neighborhood_sends_node_query_response_with_none_when_key_query_matches_no_configured_data() { - let cryptde: &dyn CryptDE = CRYPTDE_PAIR.main.as_ref(); + let cryptde: &dyn CryptDE = N_CRYPTDE_PAIR.main.as_ref(); let earning_wallet = make_wallet("earning"); let consuming_wallet = Some(make_paying_wallet(b"consuming")); let (recorder, awaiter, recording_arc) = make_recorder(); @@ -6703,7 +6692,7 @@ mod tests { addr.recipient::(); let subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::Standard( @@ -6735,7 +6724,7 @@ mod tests { context: TransmitDataMsg { endpoint: Endpoint::Key(cryptde.public_key().clone()), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: Vec::new(), }, recipient, @@ -6754,7 +6743,7 @@ mod tests { #[test] fn neighborhood_sends_node_query_response_with_result_when_key_query_matches_configured_data() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); + let cryptde = N_CRYPTDE_PAIR.main.as_ref(); let earning_wallet = make_wallet("earning"); let consuming_wallet = Some(make_paying_wallet(b"consuming")); let (recorder, awaiter, recording_arc) = make_recorder(); @@ -6764,7 +6753,7 @@ mod tests { let context = TransmitDataMsg { endpoint: Endpoint::Key(cryptde.public_key().clone()), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: Vec::new(), }; let context_a = context.clone(); @@ -6773,7 +6762,7 @@ mod tests { let addr: Addr = recorder.start(); let recipient = addr.recipient::(); let mut subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::Standard( @@ -6822,7 +6811,7 @@ mod tests { #[test] fn neighborhood_sends_node_query_response_with_none_when_ip_address_query_matches_no_configured_data( ) { - let cryptde: &dyn CryptDE = CRYPTDE_PAIR.main.as_ref(); + let cryptde: &dyn CryptDE = N_CRYPTDE_PAIR.main.as_ref(); let earning_wallet = make_wallet("earning"); let consuming_wallet = Some(make_paying_wallet(b"consuming")); let (recorder, awaiter, recording_arc) = make_recorder(); @@ -6832,7 +6821,7 @@ mod tests { let recipient: Recipient = addr.recipient::(); let subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::Standard( @@ -6864,7 +6853,7 @@ mod tests { context: TransmitDataMsg { endpoint: Endpoint::Key(cryptde.public_key().clone()), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: Vec::new(), }, recipient, @@ -6884,7 +6873,7 @@ mod tests { #[test] fn neighborhood_sends_node_query_response_with_result_when_ip_address_query_matches_configured_data( ) { - let cryptde: &dyn CryptDE = CRYPTDE_PAIR.main.as_ref(); + let cryptde: &dyn CryptDE = N_CRYPTDE_PAIR.main.as_ref(); let (recorder, awaiter, recording_arc) = make_recorder(); let node_record = make_node_record(1234, true); let another_node_record = make_node_record(2345, true); @@ -6892,7 +6881,7 @@ mod tests { let context = TransmitDataMsg { endpoint: Endpoint::Key(cryptde.public_key().clone()), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: Vec::new(), }; let context_a = context.clone(); @@ -6918,7 +6907,7 @@ mod tests { None, "neighborhood_sends_node_query_response_with_result_when_ip_address_query_matches_configured_data", ); - let mut subject = Neighborhood::new(CRYPTDE_PAIR.clone(), &config); + let mut subject = Neighborhood::new(N_CRYPTDE_PAIR.clone(), &config); subject .neighborhood_database .add_node(another_node_record_a) @@ -6957,9 +6946,12 @@ mod tests { let min_hops = Hops::TwoHops; let one_next_door_neighbor = make_node_record(3333, true); let another_next_door_neighbor = make_node_record(4444, true); - let subject_node = make_global_cryptde_node_record(5555, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A - let mut subject = - neighborhood_from_nodes(&subject_node, Some(&one_next_door_neighbor), &CRYPTDE_PAIR); + let subject_node = make_global_cryptde_node_record(5555, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let mut subject = neighborhood_from_nodes( + &subject_node, + Some(&one_next_door_neighbor), + &N_CRYPTDE_PAIR, + ); subject.min_hops = min_hops; subject @@ -6989,14 +6981,14 @@ mod tests { target_component: Component::ProxyClient, return_component_opt: Some(Component::ProxyServer), payload_size: 10000, - hostname_opt: None, + host: Host::new("booga.com", 1234), }); assert_eq!( Err(format!( "Couldn't find any routes: at least {}-hop from {} to ProxyClient at Unknown", min_hops as usize, - CRYPTDE_PAIR.main.as_ref().public_key() + N_CRYPTDE_PAIR.main.as_ref().public_key() )), result ); @@ -7007,9 +6999,9 @@ mod tests { let next_door_neighbor = make_node_record(3333, true); let exit_node = make_node_record(5, false); - let subject_node = make_global_cryptde_node_record(666, true, &CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A + let subject_node = make_global_cryptde_node_record(666, true, &N_CRYPTDE_PAIR); // 9e7p7un06eHs6frl5A let mut subject = - neighborhood_from_nodes(&subject_node, Some(&next_door_neighbor), &CRYPTDE_PAIR); + neighborhood_from_nodes(&subject_node, Some(&next_door_neighbor), &N_CRYPTDE_PAIR); subject.min_hops = Hops::TwoHops; subject @@ -7035,17 +7027,17 @@ mod tests { target_component: Component::ProxyClient, return_component_opt: Some(Component::ProxyServer), payload_size: 10000, - hostname_opt: None, + host: Host::new("host.name", 88), }); let next_door_neighbor_cryptde = CryptDENull::from(&next_door_neighbor.public_key(), TEST_DEFAULT_CHAIN); let exit_node_cryptde = CryptDENull::from(&exit_node.public_key(), TEST_DEFAULT_CHAIN); - - let hops = result.clone().unwrap().route.hops; + let response = result.clone().unwrap(); + let hops = &response.route.hops; let actual_keys: Vec = match hops.as_slice() { - [hop, exit, hop_back, origin, empty, _accounting] => vec![ - decodex::(CRYPTDE_PAIR.main.as_ref(), hop) + [hop, exit, hop_back, origin, empty] => vec![ + decodex::(N_CRYPTDE_PAIR.main.as_ref(), hop) .expect("hop") .public_key, decodex::(&next_door_neighbor_cryptde, exit) @@ -7057,11 +7049,15 @@ mod tests { decodex::(&next_door_neighbor_cryptde, origin) .expect("origin") .public_key, - decodex::(CRYPTDE_PAIR.main.as_ref(), empty) + decodex::(N_CRYPTDE_PAIR.main.as_ref(), empty) .expect("empty") .public_key, ], - l => panic!("our match is wrong, real size is {}, {:?}", l.len(), l), + l => panic!( + "our match is wrong, real size is {} instead of 5, {:?}", + l.len(), + l + ), }; let expected_public_keys = vec![ next_door_neighbor.public_key().clone(), @@ -7071,16 +7067,48 @@ mod tests { PublicKey::new(b""), ]; assert_eq!(expected_public_keys, actual_keys); + assert_eq!( + response.expected_services, + RoundTrip( + vec![ + Nothing, + Routing( + next_door_neighbor.public_key().clone(), + next_door_neighbor.earning_wallet(), + next_door_neighbor.rate_pack().clone() + ), + Exit( + exit_node.public_key().clone(), + exit_node.earning_wallet(), + exit_node.rate_pack().clone() + ), + ], + vec![ + Exit( + exit_node.public_key().clone(), + exit_node.earning_wallet(), + exit_node.rate_pack().clone() + ), + Routing( + next_door_neighbor.public_key().clone(), + next_door_neighbor.earning_wallet(), + next_door_neighbor.rate_pack().clone() + ), + Nothing, + ] + ) + ); + assert_eq!(response.host, Host::new("host.name", 88)); } fn assert_route_query_message(min_hops: Hops) { let hops = min_hops as usize; let nodes_count = hops + 1; - let root_node = make_global_cryptde_node_record(4242, true, &CRYPTDE_PAIR); + let root_node = make_global_cryptde_node_record(4242, true, &N_CRYPTDE_PAIR); let mut nodes = make_node_records(nodes_count as u16); nodes[0] = root_node; let db = linearly_connect_nodes(&nodes); - let mut subject = neighborhood_from_nodes(db.root(), nodes.get(1), &CRYPTDE_PAIR); + let mut subject = neighborhood_from_nodes(db.root(), nodes.get(1), &N_CRYPTDE_PAIR); subject.min_hops = min_hops; subject.neighborhood_database = db; @@ -7089,7 +7117,7 @@ mod tests { target_component: Component::ProxyClient, return_component_opt: Some(Component::ProxyServer), payload_size: 10000, - hostname_opt: None, + host: Host::new("booga.com", 1234), }); let assert_hops = |cryptdes: Vec, route: &[CryptData]| { @@ -7099,24 +7127,23 @@ mod tests { } }; /* - This is how the route_hops vector looks like: [C1, C2, ..., C(nodes_count), ..., C2, C1, accounting] + This is how the route_hops vector looks like: [C1, C2, ..., C(nodes_count), ..., C2, C1] Let's consider for 3-hop route ==> Nodes Count --> 4 Route Length --> 8 - Route Hops --> [C1, C2, C3, C4, C3, C2, C1, accounting] + Route Hops --> [C1, C2, C3, C4, C3, C2, C1] Over Route --> [C1, C2, C3] Back Route --> [C4, C3, C2, C1] */ - let mut route_hops = result.unwrap().route.hops; + let route_hops = result.unwrap().route.hops; let route_length = route_hops.len(); - let _accounting = route_hops.pop(); let over_route = &route_hops[..hops]; let back_route = &route_hops[hops..]; let over_cryptdes = cryptdes_from_node_records(&nodes[..hops]); let mut back_cryptdes = cryptdes_from_node_records(&nodes); back_cryptdes.reverse(); - assert_eq!(route_length, 2 * nodes_count); + assert_eq!(route_length, 2 * nodes_count - 1); assert_hops(over_cryptdes, over_route); assert_hops(back_cryptdes, back_route); } @@ -7190,16 +7217,16 @@ mod tests { target_component: Component::ProxyClient, return_component_opt: Some(Component::ProxyServer), payload_size, - hostname_opt: None, + host: Host::new("booga.com", 1234), }) .unwrap(); let (over, back) = match response.expected_services { - ExpectedServices::OneWay(_) => panic!("Expecting RoundTrip"), - ExpectedServices::RoundTrip(o, b, _) => (o[1].clone(), b[1].clone()), + OneWay(_) => panic!("Expecting RoundTrip"), + RoundTrip(o, b) => (o[1].clone(), b[1].clone()), }; let extract_key = |es: ExpectedService| match es { - ExpectedService::Routing(pk, _, _) => pk, + Routing(pk, _, _) => pk, x => panic!("Expecting Routing, found {:?}", x), }; let expected_relay_key = if a_not_b { a.clone() } else { b.clone() }; @@ -7211,7 +7238,7 @@ mod tests { #[test] fn node_record_metadata_message_is_handled_properly() { init_test_logging(); - let subject_node = make_global_cryptde_node_record(1345, true, &CRYPTDE_PAIR); + let subject_node = make_global_cryptde_node_record(1345, true, &N_CRYPTDE_PAIR); let public_key = PublicKey::from(&b"exit_node"[..]); let node_record_inputs = NodeRecordInputs { earning_wallet: make_wallet("earning"), @@ -7221,10 +7248,13 @@ mod tests { version: 0, location_opt: None, }; - let node_record = - NodeRecord::new(&public_key, CRYPTDE_PAIR.main.as_ref(), node_record_inputs); + let node_record = NodeRecord::new( + &public_key, + N_CRYPTDE_PAIR.main.as_ref(), + node_record_inputs, + ); let unreachable_host = String::from("facebook.com"); - let mut subject = neighborhood_from_nodes(&subject_node, None, &CRYPTDE_PAIR); + let mut subject = neighborhood_from_nodes(&subject_node, None, &N_CRYPTDE_PAIR); let _ = subject.neighborhood_database.add_node(node_record); let addr = subject.start(); let system = System::new("test"); @@ -7259,8 +7289,8 @@ mod tests { expected = "Neighborhood should never get ShutdownStreamMsg about non-clandestine stream" )] fn handle_stream_shutdown_complains_about_non_clandestine_message() { - let subject_node = make_global_cryptde_node_record(1345, true, &CRYPTDE_PAIR); - let mut subject = neighborhood_from_nodes(&subject_node, None, &CRYPTDE_PAIR); + let subject_node = make_global_cryptde_node_record(1345, true, &N_CRYPTDE_PAIR); + let mut subject = neighborhood_from_nodes(&subject_node, None, &N_CRYPTDE_PAIR); subject.handle_stream_shutdown_msg(StreamShutdownMsg { peer_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), @@ -7283,8 +7313,8 @@ mod tests { unrecognized_node_addr.ip_addr(), unrecognized_node_addr.ports()[0], ); - let subject_node = make_global_cryptde_node_record(1345, true, &CRYPTDE_PAIR); - let mut subject = neighborhood_from_nodes(&subject_node, None, &CRYPTDE_PAIR); + let subject_node = make_global_cryptde_node_record(1345, true, &N_CRYPTDE_PAIR); + let mut subject = neighborhood_from_nodes(&subject_node, None, &N_CRYPTDE_PAIR); let peer_actors = peer_actors_builder().hopper(hopper).build(); subject.hopper_opt = Some(peer_actors.hopper.from_hopper_client); @@ -7315,8 +7345,8 @@ mod tests { inactive_neighbor_node_addr.ip_addr(), inactive_neighbor_node_addr.ports()[0], ); - let subject_node = make_global_cryptde_node_record(1345, true, &CRYPTDE_PAIR); - let mut subject = neighborhood_from_nodes(&subject_node, None, &CRYPTDE_PAIR); + let subject_node = make_global_cryptde_node_record(1345, true, &N_CRYPTDE_PAIR); + let mut subject = neighborhood_from_nodes(&subject_node, None, &N_CRYPTDE_PAIR); subject .neighborhood_database .add_node(gossip_neighbor_node.clone()) @@ -7370,8 +7400,8 @@ mod tests { shutdown_neighbor_node_addr.ip_addr(), shutdown_neighbor_node_addr.ports()[0], ); - let subject_node = make_global_cryptde_node_record(1345, true, &CRYPTDE_PAIR); - let mut subject = neighborhood_from_nodes(&subject_node, None, &CRYPTDE_PAIR); + let subject_node = make_global_cryptde_node_record(1345, true, &N_CRYPTDE_PAIR); + let mut subject = neighborhood_from_nodes(&subject_node, None, &N_CRYPTDE_PAIR); subject .neighborhood_database .add_node(gossip_neighbor_node.clone()) @@ -7423,7 +7453,7 @@ mod tests { init_test_logging(); let system = System::new("test"); let subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::ZeroHop, @@ -7545,12 +7575,12 @@ mod tests { } #[test] - fn curate_past_neighbors_does_not_write_to_database_if_neighbors_are_same_but_order_has_changed( + fn handle_database_changes_does_not_write_to_database_if_neighbors_are_same_but_order_has_changed( ) { let mut subject = make_standard_subject(); // This mock is completely unprepared: any call to it should cause a panic let persistent_config = PersistentConfigurationMock::new(); - subject.persistent_config_opt = Some(Box::new(persistent_config)); + subject.persistent_config = Box::new(persistent_config); let neighbor_keys_before = vec![PublicKey::new(b"ABCDE"), PublicKey::new(b"FGHIJ")] .into_iter() .collect(); @@ -7558,7 +7588,7 @@ mod tests { .into_iter() .collect(); - subject.curate_past_neighbors(neighbor_keys_before, neighbor_keys_after); + subject.handle_database_changes(neighbor_keys_before, neighbor_keys_after); // No panic; therefore no attempt was made to persist: test passes! } @@ -7572,43 +7602,33 @@ mod tests { let act = |data_dir: &Path| { let mut subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_earning_wallet(make_wallet("earning_wallet")), ); - subject.data_directory = data_dir.to_path_buf(); - subject.connect_database(); + subject.persistent_config_factory = Box::new(PersistentConfigurationFactoryReal::new( + data_dir.to_path_buf(), + )); + subject.persistent_config_factory.make(); }; assert_on_initialization_with_panic_on_migration(&data_dir, &act); } fn make_standard_subject() -> Neighborhood { - let root_node = make_global_cryptde_node_record(9999, true, &CRYPTDE_PAIR); + let root_node = make_global_cryptde_node_record(9999, true, &N_CRYPTDE_PAIR); let neighbor_node = make_node_record(9998, true); - let mut subject = neighborhood_from_nodes(&root_node, Some(&neighbor_node), &CRYPTDE_PAIR); - let persistent_config = PersistentConfigurationMock::new(); - subject.persistent_config_opt = Some(Box::new(persistent_config)); + let mut subject = + neighborhood_from_nodes(&root_node, Some(&neighbor_node), &N_CRYPTDE_PAIR); + let persistent_config = PersistentConfigurationMock::new() + .rate_pack_limits_result(Ok(RatePackLimits::test_default())); + subject.persistent_config = Box::new(persistent_config); + subject.gossip_acceptor = Box::new(GossipAcceptorReal::new( + N_CRYPTDE_PAIR.main.dup(), + subject.persistent_config.as_ref(), + )); subject } - fn make_o_r_e_subject() -> (NodeRecord, NodeRecord, NodeRecord, Neighborhood) { - let mut subject = make_standard_subject(); - let o = &subject.neighborhood_database.root().clone(); - let r = &make_node_record(4567, false); - let e = &make_node_record(5678, false); - { - let db = &mut subject.neighborhood_database; - db.add_node(r.clone()).unwrap(); - db.add_node(e.clone()).unwrap(); - let mut dual_edge = |a: &NodeRecord, b: &NodeRecord| { - db.add_arbitrary_full_neighbor(a.public_key(), b.public_key()) - }; - dual_edge(o, r); - dual_edge(r, e); - } - (o.clone(), r.clone(), e.clone(), subject) - } - fn segment(nodes: &[&NodeRecord], component: &Component) -> RouteSegment { RouteSegment::new( nodes.into_iter().map(|n| n.public_key()).collect(), @@ -7627,7 +7647,7 @@ mod tests { )>, >, >, - handle_results: RefCell>, + handle_results: RefCell>>, } impl GossipAcceptor for GossipAcceptorMock { @@ -7637,9 +7657,10 @@ mod tests { agrs: Vec, gossip_source: SocketAddr, neighborhood_metadata: NeighborhoodMetadata, - ) -> GossipAcceptanceResult { + ) -> Vec { self.handle_params.lock().unwrap().push(( database.clone(), + // TODO: Figure out how to store some represntation of persistent_config agrs, gossip_source, neighborhood_metadata, @@ -7673,7 +7694,7 @@ mod tests { self } - pub fn handle_result(self, result: GossipAcceptanceResult) -> GossipAcceptorMock { + pub fn handle_result(self, result: Vec) -> GossipAcceptorMock { self.handle_results.borrow_mut().push(result); self } @@ -7754,7 +7775,7 @@ mod tests { let bootstrap_config = bc_from_nc_plus(neighborhood_config, make_wallet("earning"), None, test_name); - let mut neighborhood = Neighborhood::new(CRYPTDE_PAIR.clone(), &bootstrap_config); + let mut neighborhood = Neighborhood::new(N_CRYPTDE_PAIR.clone(), &bootstrap_config); let (node_to_ui_recipient, _) = make_node_to_ui_recipient(); neighborhood.node_to_ui_recipient_opt = Some(node_to_ui_recipient); @@ -7769,7 +7790,7 @@ mod tests { ) -> Option { let system = System::new("test"); let mut subject = Neighborhood::new( - CRYPTDE_PAIR.clone(), + N_CRYPTDE_PAIR.clone(), &bc_from_nc_plus( NeighborhoodConfig { mode: NeighborhoodMode::ConsumeOnly(vec![make_node_descriptor(make_ip(1))]), @@ -7808,12 +7829,17 @@ mod tests { } fn make_neighborhood_with_linearly_connected_nodes(nodes_count: u16) -> Neighborhood { - let root_node = make_global_cryptde_node_record(4242, true, &CRYPTDE_PAIR); + let root_node = make_global_cryptde_node_record(4242, true, &N_CRYPTDE_PAIR); let mut nodes = make_node_records(nodes_count); nodes[0] = root_node; let db = linearly_connect_nodes(&nodes); - let mut neighborhood = neighborhood_from_nodes(db.root(), nodes.get(1), &CRYPTDE_PAIR); + let mut neighborhood = neighborhood_from_nodes(db.root(), nodes.get(1), &N_CRYPTDE_PAIR); neighborhood.neighborhood_database = db; + let persistent_config_mock = PersistentConfigurationMock::new() + .rate_pack_limits_result(Ok(RatePackLimits::test_default())); + let gossip_acceptor = + GossipAcceptorReal::new(N_CRYPTDE_PAIR.main.dup(), &persistent_config_mock); + neighborhood.gossip_acceptor = Box::new(gossip_acceptor); neighborhood } diff --git a/node/src/neighborhood/neighborhood_database.rs b/node/src/neighborhood/neighborhood_database.rs index c8bd22212..6214ca1ce 100644 --- a/node/src/neighborhood/neighborhood_database.rs +++ b/node/src/neighborhood/neighborhood_database.rs @@ -313,7 +313,7 @@ impl NeighborhoodDatabase { country_code, }), public_key: public_key.clone(), - node_addr: nr.node_addr_opt(), + node_addr_opt: nr.node_addr_opt(), known_source: public_key == self.root().public_key(), known_target: false, is_present: true, @@ -323,7 +323,7 @@ impl NeighborhoodDatabase { node_renderables.push(NodeRenderable { inner: None, public_key: k.clone(), - node_addr: None, + node_addr_opt: None, known_source: false, known_target: false, is_present: false, diff --git a/node/src/neighborhood/node_record.rs b/node/src/neighborhood/node_record.rs index a3105f1df..1e6a84cbb 100644 --- a/node/src/neighborhood/node_record.rs +++ b/node/src/neighborhood/node_record.rs @@ -10,11 +10,13 @@ use crate::sub_lib::neighborhood::{NodeDescriptor, RatePack}; use crate::sub_lib::node_addr::NodeAddr; use crate::sub_lib::utils::time_t_timestamp; use crate::sub_lib::wallet::Wallet; +use itertools::Itertools; use masq_lib::blockchains::chains::Chain; use serde_derive::{Deserialize, Serialize}; use std::collections::btree_set::BTreeSet; use std::collections::HashSet; use std::convert::TryFrom; +use std::fmt::{Display, Formatter}; //TODO #584 create special serializer for NodeRecordInner_0v1 #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] @@ -49,6 +51,69 @@ impl TryFrom<&GossipNodeRecord> for NodeRecordInner_0v1 { } } +impl Display for NodeRecordInner_0v1 { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} {} {} {} {} {} {}", + self.behavior_string(), + self.version_string(), + self.country_code_string(), + self.public_key_string(), + self.wallet_string(), + self.rate_pack_string(), + self.neighbors_string() + ) + } +} + +impl NodeRecordInner_0v1 { + pub fn behavior_string(&self) -> String { + let accepts_connections = if self.accepts_connections { "A" } else { "a" }; + let routes_data = if self.routes_data { "R" } else { "r" }; + format!("{}{}", accepts_connections, routes_data) + } + + pub fn version_string(&self) -> String { + format!("v{}", self.version) + } + + pub fn country_code_string(&self) -> String { + match &self.country_code_opt { + Some(cc) => cc.clone(), + None => "ZZ".to_string(), + } + } + + pub fn public_key_string(&self) -> String { + let mut public_key = self.public_key.to_string(); + public_key.truncate(8); + public_key + } + + pub fn wallet_string(&self) -> String { + self.earning_wallet.to_string() + } + + pub fn rate_pack_string(&self) -> String { + self.rate_pack.rate_pack_parameter() + } + + pub fn neighbors_string(&self) -> String { + let filling = self + .neighbors + .iter() + .map(|it| { + let mut s = it.to_string(); + s.truncate(8); + s + }) + .collect_vec() + .join(", "); + format!("[{}]", filling) + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub enum NodeRecordError { SelfNeighborAttempt(PublicKey), @@ -427,6 +492,51 @@ mod tests { assert_eq!(actual_node_record, expected_node_record); } + #[test] + fn node_record_inner_0v1_display_works_1() { + let mut subject = make_node_record(1234, true); + subject.inner.accepts_connections = false; + subject.inner.routes_data = false; + subject.inner.version = 19; + subject.inner.country_code_opt = None; + subject.inner.earning_wallet = + Wallet::from_str("0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee").unwrap(); + subject.inner.rate_pack = RatePack::new(100, 200, 300, 400); + let neighbor1 = PublicKey::new(&b"fiddle"[..]); + let neighbor2 = PublicKey::new(&b"diffle"[..]); + subject.inner.neighbors = vec![neighbor1, neighbor2] + .into_iter() + .collect::>(); + + let result = subject.inner.to_string(); + + assert_eq!( + result, + "ar v19 ZZ AQIDBA 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee 100|200|300|400 [ZGlmZmxl, ZmlkZGxl]".to_string() + ) + } + + #[test] + fn node_record_inner_0v1_display_works_2() { + let mut subject = make_node_record(2345, true); + subject.inner.accepts_connections = true; + subject.inner.routes_data = true; + subject.inner.version = 91; + subject.inner.country_code_opt = Some("US".to_string()); + subject.inner.earning_wallet = + Wallet::from_str("0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee00").unwrap(); + subject.inner.rate_pack = RatePack::new(400, 300, 200, 100); + subject.inner.neighbors = vec![].into_iter().collect::>(); + + let result = subject.inner.to_string(); + + assert_eq!( + result, + "AR v91 US AgMEBQ 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee00 400|300|200|100 []" + .to_string() + ) + } + #[test] fn set_node_addr_works_once_but_not_twice() { let mut subject = make_node_record(1234, false); diff --git a/node/src/neighborhood/overall_connection_status.rs b/node/src/neighborhood/overall_connection_status.rs index abb533f8c..57921ec4a 100644 --- a/node/src/neighborhood/overall_connection_status.rs +++ b/node/src/neighborhood/overall_connection_status.rs @@ -73,7 +73,7 @@ impl ConnectionProgress { let new_stage = usize::try_from(&connection_stage); if let (Ok(current_stage_num), Ok(new_stage_num)) = (current_stage, new_stage) { - if new_stage_num != current_stage_num + 1 { + if (new_stage_num != current_stage_num) && (new_stage_num != current_stage_num + 1) { panic!( "Can't update the stage from {:?} to {:?}", self.connection_stage, connection_stage @@ -342,6 +342,82 @@ mod tests { use masq_lib::test_utils::logging::{init_test_logging, TestLogHandler}; use masq_lib::ui_gateway::MessageTarget; + #[test] + fn update_stage_tolerates_advancement() { + let cases = vec![ + ( + ConnectionStage::StageZero, + ConnectionStage::TcpConnectionEstablished, + ), + ( + ConnectionStage::TcpConnectionEstablished, + ConnectionStage::NeighborshipEstablished, + ), + ( + ConnectionStage::StageZero, + ConnectionStage::Failed(TcpConnectionFailed), + ), + ( + ConnectionStage::TcpConnectionEstablished, + ConnectionStage::Failed(PassLoopFound), + ), + ( + ConnectionStage::NeighborshipEstablished, + ConnectionStage::Failed(NoGossipResponseReceived), + ), + ]; + cases.into_iter().for_each(|(from_stage, to_stage)| { + let mut subject = ConnectionProgress { + initial_node_descriptor: make_node_descriptor(make_ip(1)), + current_peer_addr: make_ip(1), + connection_stage: from_stage, + }; + + subject.update_stage( + &Logger::new("update_stage_tolerates_advancement"), + to_stage.clone(), + ); + + assert_eq!(subject.connection_stage, to_stage); + }) + } + + #[test] + #[should_panic(expected = "Can't update the stage from StageZero to NeighborshipEstablished")] + fn update_stage_does_not_tolerate_skipping() { + let mut subject = ConnectionProgress { + initial_node_descriptor: make_node_descriptor(make_ip(1)), + current_peer_addr: make_ip(1), + connection_stage: ConnectionStage::StageZero, + }; + + subject.update_stage( + &Logger::new("update_stage_does_not_tolerate_skipping"), + ConnectionStage::NeighborshipEstablished, + ); + } + + #[test] + fn update_stage_tolerates_stasis() { + let cases = vec![ + ConnectionStage::StageZero, + ConnectionStage::TcpConnectionEstablished, + ConnectionStage::NeighborshipEstablished, + ConnectionStage::Failed(TcpConnectionFailed), + ]; + cases.into_iter().for_each(|stage| { + let mut subject = ConnectionProgress { + initial_node_descriptor: make_node_descriptor(make_ip(1)), + current_peer_addr: make_ip(1), + connection_stage: stage.clone(), + }; + + subject.update_stage(&Logger::new("update_stage_tolerates_stasis"), stage.clone()); + + assert_eq!(subject.connection_stage, stage); + }) + } + #[test] #[should_panic( expected = "Unable to receive node addr for the descriptor NodeDescriptor { blockchain: EthRopsten, encryption_public_key: 0x000000, node_addr_opt: None }" diff --git a/node/src/node_configurator/mod.rs b/node/src/node_configurator/mod.rs index 5738fe3b3..c549bfd5e 100644 --- a/node/src/node_configurator/mod.rs +++ b/node/src/node_configurator/mod.rs @@ -233,8 +233,7 @@ pub fn determine_user_specific_data( Box::new(CommandLineVcl::new(args.to_vec())), ); /* We create multiconfig to retrieve chain, real-user, data-directory and config file, to establish ConfigVcl */ - let first_multi_config = - make_new_multi_config(app, vec![orientation_args]).expect("expected MultiConfig"); + let first_multi_config = make_new_multi_config(app, vec![orientation_args])?; let initialization_data = config_file_data_dir_real_user_chain_from_mc(dirs_wrapper, first_multi_config); @@ -350,7 +349,7 @@ impl Default for DirsWrapperReal { mod tests { use super::*; use crate::node_test_utils::DirsWrapperMock; - use crate::test_utils::ArgsBuilder; + use crate::test_utils::{assert_string_contains, ArgsBuilder}; use masq_lib::shared_schema::{config_file_arg, data_directory_arg, DATA_DIRECTORY_HELP}; use masq_lib::test_utils::environment_guard::EnvironmentGuard; use masq_lib::test_utils::utils::ensure_node_home_directory_exists; @@ -389,10 +388,33 @@ mod tests { } #[test] - fn determine_config_file_path_finds_path_in_args() { + fn determine_user_specific_data_detects_bad_parameter() { + let args = ArgsBuilder::new().param("--booga-booga", "bad-parameter"); + let args_vec: Vec = args.into(); + let app = determine_config_file_path_app(); + + let result = + determine_user_specific_data(&DirsWrapperReal::default(), &app, args_vec.as_slice()) + .unwrap_err(); + + let param_error = &result.param_errors[0]; + assert_eq!(param_error.parameter, ""); + assert_string_contains(¶m_error.reason, "Unfamiliar message: "); + assert_string_contains(¶m_error.reason, "error:"); + assert_string_contains(¶m_error.reason, "Found argument '"); + assert_string_contains(¶m_error.reason, "--booga-booga"); + assert_string_contains( + ¶m_error.reason, + "' which wasn't expected, or isn't valid in this context", + ); + assert_eq!(result.param_errors.len(), 1); + } + + #[test] + fn determine_user_specific_data_finds_path_in_args() { let data_directory = ensure_node_home_directory_exists( "node_configurator", - "determine_config_file_path_finds_path_in_args", + "determine_user_specific_data_finds_path_in_args", ); let _guard = EnvironmentGuard::new(); let args = ArgsBuilder::new() @@ -432,10 +454,10 @@ mod tests { } #[test] - fn determine_config_file_path_finds_path_in_environment() { + fn determine_user_specific_data_finds_path_in_environment() { let data_directory = ensure_node_home_directory_exists( "node_configurator", - "determine_config_file_path_finds_path_in_environment", + "determine_user_specific_data_finds_path_in_environment", ); let _guard = EnvironmentGuard::new(); let args = ArgsBuilder::new(); @@ -476,7 +498,7 @@ mod tests { #[cfg(not(target_os = "windows"))] #[test] - fn determine_config_file_path_ignores_data_dir_if_config_file_has_root() { + fn determine_user_specific_data_ignores_data_dir_if_config_file_has_root() { let _guard = EnvironmentGuard::new(); let args = ArgsBuilder::new() .param("--data-directory", "data-dir") @@ -498,7 +520,7 @@ mod tests { #[cfg(target_os = "windows")] #[test] - fn determine_config_file_path_ignores_data_dir_if_config_file_has_separator_root() { + fn determine_user_specific_data_ignores_data_dir_if_config_file_has_separator_root() { let _guard = EnvironmentGuard::new(); let args = ArgsBuilder::new() .param("--data-directory", "data-dir") @@ -522,7 +544,7 @@ mod tests { #[cfg(target_os = "windows")] #[test] - fn determine_config_file_path_ignores_data_dir_if_config_file_has_drive_root() { + fn determine_user_specific_data_ignores_data_dir_if_config_file_has_drive_root() { let _guard = EnvironmentGuard::new(); let args = ArgsBuilder::new() .param("--data-directory", "data-dir") @@ -545,7 +567,7 @@ mod tests { #[cfg(target_os = "windows")] #[test] - fn determine_config_file_path_ignores_data_dir_if_config_file_has_network_root() { + fn determine_user_specific_data_ignores_data_dir_if_config_file_has_network_root() { let _guard = EnvironmentGuard::new(); let args = ArgsBuilder::new() .param("--data-directory", "data-dir") @@ -568,7 +590,7 @@ mod tests { #[cfg(target_os = "windows")] #[test] - fn determine_config_file_path_ignores_data_dir_if_config_file_has_drive_letter_but_no_separator( + fn determine_user_specific_data_ignores_data_dir_if_config_file_has_drive_letter_but_no_separator( ) { let _guard = EnvironmentGuard::new(); let args = ArgsBuilder::new() diff --git a/node/src/node_configurator/node_configurator_standard.rs b/node/src/node_configurator/node_configurator_standard.rs index 4e72a6716..da9795df0 100644 --- a/node/src/node_configurator/node_configurator_standard.rs +++ b/node/src/node_configurator/node_configurator_standard.rs @@ -6,7 +6,7 @@ use crate::node_configurator::{ConfigInitializationData, DirsWrapperReal}; use masq_lib::crash_point::CrashPoint; use masq_lib::logger::Logger; use masq_lib::multi_config::{MultiConfig, VirtualCommandLine}; -use masq_lib::shared_schema::ConfiguratorError; +use masq_lib::shared_schema::{ConfiguratorError, OnOff}; use masq_lib::utils::NeighborhoodModeLight; use std::net::SocketAddr; use std::net::{IpAddr, Ipv4Addr}; @@ -95,7 +95,10 @@ impl NodeConfigurator for NodeConfiguratorStandardUnprivileg )?; configure_database(&unprivileged_config, persistent_config.as_mut())?; let cryptde_pair = if multi_config.occurrences_of("fake-public-key") == 0 { + let new_public_key = value_m!(multi_config, "new-public-key", OnOff); + configure_cryptdes( + new_public_key, persistent_config.as_mut(), &unprivileged_config.db_password_opt, ) @@ -300,6 +303,15 @@ pub fn privileged_parse_args( None => vec![SocketAddr::new(IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)), 53)], }; + privileged_config.new_public_key_opt = match value_m!(multi_config, "new-public-key", String) { + Some(value) => match value.as_str() { + "on" => Some(true), + "off" => Some(false), + _ => panic!("Bad clap validation for new-public-key: {}", value), + }, + None => None, + }; + privileged_config.log_level = value_m!(multi_config, "log-level", LevelFilter).unwrap_or(LevelFilter::Warn); @@ -345,17 +357,30 @@ fn configure_database( } fn configure_cryptdes( + new_public_key: Option, persistent_config: &mut dyn PersistentConfiguration, db_password_opt: &Option, ) -> CryptDEPair { let cryptde_pair = if let Some(db_password) = db_password_opt { let chain = Chain::from(persistent_config.chain_name().as_str()); - let main_result = persistent_config.cryptde(db_password); + let main_result = match new_public_key { + None | Some(OnOff::Off) => persistent_config.cryptde(db_password), + Some(OnOff::On) => { + let main_cryptde: Box = Box::new(CryptDEReal::new(chain)); + persistent_config + .set_cryptde(main_cryptde.as_ref(), db_password) + .expect("Failed to set cryptde"); + Ok(Some(main_cryptde)) + } + }; match main_result { Ok(Some(last_main_cryptde)) => { CryptDEPair::new(last_main_cryptde, Box::new(CryptDEReal::new(chain))) } Ok(None) => { + if new_public_key == Some(OnOff::Off) { + panic!("--new-public-key off: Cannot reestablish old public key: no old public key available"); + } let main_cryptde: Box = Box::new(CryptDEReal::new(chain)); persistent_config .set_cryptde(main_cryptde.as_ref(), db_password) @@ -366,6 +391,9 @@ fn configure_cryptdes( Err(e) => panic!("Could not read last cryptde from database: {:?}", e), } } else { + if new_public_key == Some(OnOff::Off) { + panic!("--new-public-key off: Cannot reestablish old public key: no --db-password provided"); + } let chain = Chain::from(persistent_config.chain_name().as_str()); let main_cryptde: Box = Box::new(CryptDEReal::new(chain)); let alias_cryptde: Box = Box::new(CryptDEReal::new(chain)); @@ -534,6 +562,35 @@ mod tests { ); } + #[test] + #[should_panic( + expected = "--new-public-key off: Cannot reestablish old public key: no --db-password provided" + )] + fn node_configurator_standard_unprivileged_complains_if_no_password_and_new_public_key_off() { + let home_dir = ensure_node_home_directory_exists( + "node_configurator_standard", + "node_configurator_standard_unprivileged_complains_if_no_password_and_new_public_key_off", + ); + let multi_config = make_simplified_multi_config([ + "--chain", + "eth-mainnet", + "--new-public-key", + "off", + "--ip", + "1.2.3.4", + "--blockchain-service-url", + "https://booga.com", + ]); + let mut privileged_config = BootstrapperConfig::default(); + privileged_config.data_directory = home_dir; + let subject = NodeConfiguratorStandardUnprivileged { + privileged_config, + logger: Logger::new("test"), + }; + + let _ = subject.configure(&multi_config).unwrap(); + } + #[test] fn node_configurator_standard_unprivileged_handles_fake_public_key() { let home_dir = ensure_node_home_directory_exists( @@ -644,7 +701,17 @@ mod tests { } #[test] - fn configure_cryptdes_handles_missing_password_with_uninitialized_cryptdes() { + #[should_panic( + expected = "--new-public-key off: Cannot reestablish old public key: no --db-password provided" + )] + fn configure_cryptdes_handles_missing_password_with_uninitialized_cryptdes_and_npk_off() { + let mut persistent_config = PersistentConfigurationMock::new(); + + configure_cryptdes(Some(OnOff::Off), &mut persistent_config, &None); + } + + #[test] + fn configure_cryptdes_handles_missing_password_with_uninitialized_cryptdes_and_npk_on() { let cryptde_params_arc = Arc::new(Mutex::new(vec![])); let set_cryptde_params_arc = Arc::new(Mutex::new(vec![])); let mut persistent_config = PersistentConfigurationMock::new() @@ -652,7 +719,7 @@ mod tests { .set_cryptde_params(&set_cryptde_params_arc) .chain_name_result(TEST_DEFAULT_CHAIN.to_string()); - let _result = configure_cryptdes(&mut persistent_config, &None); + let _result = configure_cryptdes(Some(OnOff::On), &mut persistent_config, &None); let cryptde_params = cryptde_params_arc.lock().unwrap(); assert_eq!(cryptde_params.len(), 0); @@ -661,7 +728,7 @@ mod tests { } #[test] - fn configure_cryptdes_handles_missing_last_cryptde() { + fn configure_cryptdes_handles_missing_last_cryptde_with_no_npk_param() { let cryptde_params_arc = Arc::new(Mutex::new(vec![])); let set_cryptde_params_arc = Arc::new(Mutex::new(vec![])); let mut persistent_config = PersistentConfigurationMock::new() @@ -671,7 +738,11 @@ mod tests { .set_cryptde_params(&set_cryptde_params_arc) .set_cryptde_result(Ok(())); - let result = configure_cryptdes(&mut persistent_config, &Some("db_password".to_string())); + let result = configure_cryptdes( + None, + &mut persistent_config, + &Some("db_password".to_string()), + ); let cryptde_params = cryptde_params_arc.lock().unwrap(); assert_eq!(*cryptde_params, vec!["db_password".to_string()]); @@ -681,6 +752,44 @@ mod tests { assert_eq!(call.1, "db_password".to_string()); } + #[test] + #[should_panic( + expected = "--new-public-key off: Cannot reestablish old public key: no old public key available" + )] + fn configure_cryptdes_handles_missing_last_cryptde_with_npk_off() { + let cryptde_params_arc = Arc::new(Mutex::new(vec![])); + let mut persistent_config = PersistentConfigurationMock::new() + .cryptde_params(&cryptde_params_arc) + .chain_name_result(TEST_DEFAULT_CHAIN.to_string()) + .cryptde_result(Ok(None)); + + configure_cryptdes( + Some(OnOff::Off), + &mut persistent_config, + &Some("db_password".to_string()), + ); + } + + #[test] + fn configure_cryptdes_handles_missing_last_cryptde_with_npk_on() { + let set_cryptde_params_arc = Arc::new(Mutex::new(vec![])); + let mut persistent_config = PersistentConfigurationMock::new() + .chain_name_result(TEST_DEFAULT_CHAIN.to_string()) + .set_cryptde_params(&set_cryptde_params_arc) + .set_cryptde_result(Ok(())); + + let result = configure_cryptdes( + Some(OnOff::On), + &mut persistent_config, + &Some("db_password".to_string()), + ); + + let set_cryptde_params = set_cryptde_params_arc.lock().unwrap(); + let call = &set_cryptde_params[0]; + assert_eq!(call.0.public_key(), result.main.public_key()); + assert_eq!(call.1, "db_password".to_string()); + } + #[test] #[should_panic(expected = "Could not read last cryptde from database: NotPresent")] fn configure_cryptdes_panics_if_database_throws_error() { @@ -689,11 +798,15 @@ mod tests { .chain_name_result(TEST_DEFAULT_CHAIN.to_string()) .cryptde_result(Err(PersistentConfigError::NotPresent)); - let _ = configure_cryptdes(&mut persistent_config, &Some("db_password".to_string())); + let _ = configure_cryptdes( + None, + &mut persistent_config, + &Some("db_password".to_string()), + ); } #[test] - fn configure_cryptdes_handles_populated_database() { + fn configure_cryptdes_handles_populated_database_with_no_npk_param() { let _guard = EnvironmentGuard::new(); let stored_main_cryptde_box = Box::new(CryptDEReal::new(TEST_DEFAULT_CHAIN)); let cryptde_params_arc = Arc::new(Mutex::new(vec![])); @@ -702,7 +815,11 @@ mod tests { .chain_name_result(TEST_DEFAULT_CHAIN.to_string()) .cryptde_result(Ok(Some(stored_main_cryptde_box.dup()))); - let result = configure_cryptdes(&mut persistent_config, &Some("db_password".to_string())); + let result = configure_cryptdes( + None, + &mut persistent_config, + &Some("db_password".to_string()), + ); assert_eq!( result.main.public_key(), @@ -712,6 +829,59 @@ mod tests { assert_eq!(*cryptde_params, vec!["db_password".to_string()]); } + #[test] + fn configure_cryptdes_handles_populated_database_with_npk_off() { + let _guard = EnvironmentGuard::new(); + let stored_main_cryptde_box = Box::new(CryptDEReal::new(TEST_DEFAULT_CHAIN)); + let cryptde_params_arc = Arc::new(Mutex::new(vec![])); + let mut persistent_config = PersistentConfigurationMock::new() + .cryptde_params(&cryptde_params_arc) + .chain_name_result(TEST_DEFAULT_CHAIN.to_string()) + .cryptde_result(Ok(Some(stored_main_cryptde_box.dup()))); + + let result = configure_cryptdes( + Some(OnOff::Off), + &mut persistent_config, + &Some("db_password".to_string()), + ); + + assert_eq!( + result.main.public_key(), + stored_main_cryptde_box.public_key() + ); + let cryptde_params = cryptde_params_arc.lock().unwrap(); + assert_eq!(*cryptde_params, vec!["db_password".to_string()]); + } + + #[test] + fn configure_cryptdes_handles_populated_database_with_npk_on() { + let _guard = EnvironmentGuard::new(); + let stored_main_cryptde_box = Box::new(CryptDEReal::new(TEST_DEFAULT_CHAIN)); + let set_cryptde_params_arc = Arc::new(Mutex::new(vec![])); + let mut persistent_config = PersistentConfigurationMock::new() + .chain_name_result(TEST_DEFAULT_CHAIN.to_string()) + .set_cryptde_params(&set_cryptde_params_arc) + .set_cryptde_result(Ok(())); + + let result = configure_cryptdes( + Some(OnOff::On), + &mut persistent_config, + &Some("db_password".to_string()), + ); + + assert_ne!( + result.main.public_key(), + stored_main_cryptde_box.public_key() + ); + let set_cryptde_params = set_cryptde_params_arc.lock().unwrap(); + assert_eq!( + set_cryptde_params[0].0.public_key(), + result.main.public_key() + ); + assert_eq!(set_cryptde_params[0].1, "db_password".to_string()); + assert_eq!(set_cryptde_params.len(), 1); + } + fn make_default_cli_params() -> ArgsBuilder { ArgsBuilder::new().param("--ip", "1.2.3.4") } @@ -850,6 +1020,7 @@ mod tests { "--consuming-private-key", "ABCDEF01ABCDEF01ABCDEF01ABCDEF01ABCDEF01ABCDEF01ABCDEF01ABCDEF01", ) + .param("--new-public-key", "on") .param("--real-user", "999:999:/home/booga") .param("--chain", "polygon-amoy"); let mut config = BootstrapperConfig::new(); @@ -883,12 +1054,28 @@ mod tests { None, ); assert_eq!(config.data_directory, home_dir); + assert_eq!(config.new_public_key_opt, Some(true)); assert_eq!( config.real_user, RealUser::new(Some(999), Some(999), Some(PathBuf::from("/home/booga"))) ); } + #[test] + fn privileged_parse_args_works_with_on_off_parameters() { + let _guard = EnvironmentGuard::new(); + running_test(); + let args = ArgsBuilder::new().param("--new-public-key", "on"); + let mut config = BootstrapperConfig::new(); + let vcls: Vec> = + vec![Box::new(CommandLineVcl::new(args.into()))]; + let multi_config = make_new_multi_config(&app_node(), vcls).unwrap(); + + privileged_parse_args(&DirsWrapperReal::default(), &multi_config, &mut config).unwrap(); + + assert_eq!(config.new_public_key_opt, Some(true)); + } + #[test] fn privileged_parse_args_creates_configuration_with_defaults() { let _guard = EnvironmentGuard::new(); @@ -908,6 +1095,7 @@ mod tests { ); assert_eq!(config.crash_point, CrashPoint::None); assert_eq!(config.ui_gateway_config.ui_port, DEFAULT_UI_PORT); + assert_eq!(config.new_public_key_opt, None); assert_eq!( config.real_user, RealUser::new(None, None, None).populate(&DirsWrapperReal::default()) @@ -963,6 +1151,7 @@ mod tests { ); assert_eq!(config.crash_point, CrashPoint::None); assert_eq!(config.ui_gateway_config.ui_port, DEFAULT_UI_PORT); + assert_eq!(config.new_public_key_opt, None); assert_eq!( config.real_user, RealUser::new(None, None, None).populate(&DirsWrapperReal::default()) @@ -1023,6 +1212,9 @@ mod tests { config_file .write_all(b"neighborhood-mode = \"zero-hop\"\n") .unwrap(); + config_file + .write_all(b"new-public-key = \"off\"\n") + .unwrap(); config_file .write_all(b"payment-thresholds = \"3333|55|33|646|999|999\"\n") .unwrap(); @@ -1623,9 +1815,6 @@ mod tests { assert_eq!(config.blockchain_bridge_config.gas_price, 1); } - #[should_panic( - expected = "expected MultiConfig: ConfiguratorError { param_errors: [ParamError { parameter: \"gas-price\", reason: \"Invalid value: unleaded\" }] }" - )] #[test] fn server_initializer_collected_params_rejects_invalid_gas_price() { let _guard = EnvironmentGuard::new(); diff --git a/node/src/node_configurator/unprivileged_parse_args_configuration.rs b/node/src/node_configurator/unprivileged_parse_args_configuration.rs index 313bf3048..2c674eba1 100644 --- a/node/src/node_configurator/unprivileged_parse_args_configuration.rs +++ b/node/src/node_configurator/unprivileged_parse_args_configuration.rs @@ -575,14 +575,82 @@ fn configure_rate_pack( multi_config: &MultiConfig, persist_config: &mut dyn PersistentConfiguration, ) -> Result { - process_combined_params( + let check_min_and_max = |candidate: u64, + min: u64, + max: u64, + name: &str, + error: ConfiguratorError| + -> ConfiguratorError { + let mut result = error; + if candidate < min { + result = result.another_required( + "rate-pack", + &format!( + "Value of {} ({}) is below the minimum allowed ({})", + name, candidate, min + ), + ); + } else if candidate > max { + result = result.another_required( + "rate-pack", + &format!( + "Value of {} ({}) is above the maximum allowed ({})", + name, candidate, max + ), + ); + } + result + }; + match process_combined_params( "rate-pack", multi_config, persist_config, |str: &str| RatePack::try_from(str), |pc: &dyn PersistentConfiguration| pc.rate_pack(), |pc: &mut dyn PersistentConfiguration, rate_pack| pc.set_rate_pack(rate_pack), - ) + ) { + Ok(rate_pack) => { + let rate_pack_limits = match persist_config.rate_pack_limits() { + Ok(rpl) => rpl, + Err(e) => return Err(e.into_configurator_error("rate-pack")), + }; + let mut error = ConfiguratorError::new(vec![]); + error = check_min_and_max( + rate_pack.routing_byte_rate, + rate_pack_limits.lo.routing_byte_rate, + rate_pack_limits.hi.routing_byte_rate, + "routing_byte_rate", + error, + ); + error = check_min_and_max( + rate_pack.routing_service_rate, + rate_pack_limits.lo.routing_service_rate, + rate_pack_limits.hi.routing_service_rate, + "routing_service_rate", + error, + ); + error = check_min_and_max( + rate_pack.exit_byte_rate, + rate_pack_limits.lo.exit_byte_rate, + rate_pack_limits.hi.exit_byte_rate, + "exit_byte_rate", + error, + ); + error = check_min_and_max( + rate_pack.exit_service_rate, + rate_pack_limits.lo.exit_service_rate, + rate_pack_limits.hi.exit_service_rate, + "exit_service_rate", + error, + ); + if !error.is_empty() { + Err(error) + } else { + Ok(rate_pack) + } + } + Err(e) => Err(e), + } } fn process_combined_params<'a, T: PartialEq, C1, C2>( @@ -645,7 +713,7 @@ mod tests { use crate::db_config::persistent_configuration::PersistentConfigurationReal; use crate::sub_lib::accountant::DEFAULT_PAYMENT_THRESHOLDS; use crate::sub_lib::cryptde::{PlainData, PublicKey}; - use crate::sub_lib::neighborhood::{Hops, DEFAULT_RATE_PACK}; + use crate::sub_lib::neighborhood::{Hops, RatePackLimits, DEFAULT_RATE_PACK}; use crate::sub_lib::utils::make_new_multi_config; use crate::sub_lib::wallet::Wallet; use crate::test_utils::neighborhood_test_utils::MIN_HOPS_FOR_TEST; @@ -2466,6 +2534,76 @@ mod tests { assert_eq!(result, expected_rate_pack) } + #[test] + fn configure_rate_pack_complains_when_minimums_are_transgressed() { + let mut persistent_config = PersistentConfigurationMock::new() + .rate_pack_result(Ok(RatePack::new(0, 0, 0, 0))) + .set_rate_pack_result(Ok(())) + .rate_pack_limits_result(Ok(RatePackLimits::new( + RatePack::new(5, 5, 5, 5), + RatePack::new(7, 7, 7, 7), + ))); + + let result = configure_rate_pack( + &make_simplified_multi_config(["--rate-pack", "4|4|4|4"]), + &mut persistent_config, + ); + + let expected_error = ConfiguratorError::new(vec![]) + .another_required( + "rate-pack", + "Value of routing_byte_rate (4) is below the minimum allowed (5)", + ) + .another_required( + "rate-pack", + "Value of routing_service_rate (4) is below the minimum allowed (5)", + ) + .another_required( + "rate-pack", + "Value of exit_byte_rate (4) is below the minimum allowed (5)", + ) + .another_required( + "rate-pack", + "Value of exit_service_rate (4) is below the minimum allowed (5)", + ); + assert_eq!(result, Err(expected_error)); + } + + #[test] + fn configure_rate_pack_complains_when_maximums_are_transgressed() { + let mut persistent_config = PersistentConfigurationMock::new() + .rate_pack_result(Ok(RatePack::new(0, 0, 0, 0))) + .set_rate_pack_result(Ok(())) + .rate_pack_limits_result(Ok(RatePackLimits::new( + RatePack::new(5, 5, 5, 5), + RatePack::new(7, 7, 7, 7), + ))); + + let result = configure_rate_pack( + &make_simplified_multi_config(["--rate-pack", "8|8|8|8"]), + &mut persistent_config, + ); + + let expected_error = ConfiguratorError::new(vec![]) + .another_required( + "rate-pack", + "Value of routing_byte_rate (8) is above the maximum allowed (7)", + ) + .another_required( + "rate-pack", + "Value of routing_service_rate (8) is above the maximum allowed (7)", + ) + .another_required( + "rate-pack", + "Value of exit_byte_rate (8) is above the maximum allowed (7)", + ) + .another_required( + "rate-pack", + "Value of exit_service_rate (8) is above the maximum allowed (7)", + ); + assert_eq!(result, Err(expected_error)); + } + #[test] fn compute_mapping_protocol_returns_saved_value_if_nothing_supplied() { let multi_config = make_new_multi_config( @@ -2706,6 +2844,10 @@ mod tests { .past_neighbors_result(past_neighbors_result) .mapping_protocol_result(Ok(Some(AutomapProtocol::Pcp))) .rate_pack_result(Ok(rate_pack)) + .rate_pack_limits_result(Ok(RatePackLimits::new( + RatePack::new(u64::MIN, u64::MIN, u64::MIN, u64::MIN), + RatePack::new(u64::MAX, u64::MAX, u64::MAX, u64::MAX), + ))) .min_hops_result(Ok(min_hops)) } } diff --git a/node/src/proxy_client/mod.rs b/node/src/proxy_client/mod.rs index aa0f33010..c17daaf16 100644 --- a/node/src/proxy_client/mod.rs +++ b/node/src/proxy_client/mod.rs @@ -613,7 +613,7 @@ mod tests { sequence_number: 0, last_data: false, }, - target_hostname: Some(String::from("target.hostname.com")), + target_hostname: String::from("target.hostname.com"), target_port: 1234, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(&b"originator_public_key"[..]), @@ -622,7 +622,7 @@ mod tests { let package = ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("consuming")), - route_to_proxy_client(&cryptde.public_key(), cryptde), + route_to_proxy_client(&cryptde.public_key(), cryptde, false), request, 0, ); @@ -754,7 +754,7 @@ mod tests { sequence_number: 0, last_data: false, }, - target_hostname: None, + target_hostname: "booga.com".to_string(), target_port: 0, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(&b"originator"[..]), @@ -813,7 +813,7 @@ mod tests { sequence_number: 0, last_data: false, }, - target_hostname: None, + target_hostname: "booga.com".to_string(), target_port: 0, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(&b"originator"[..]), @@ -870,7 +870,7 @@ mod tests { sequence_number: 0, last_data: false, }, - target_hostname: None, + target_hostname: "booga.com".to_string(), target_port: 0, protocol: ProxyProtocol::HTTP, originator_public_key: alias_cryptde.public_key().clone(), @@ -1226,7 +1226,7 @@ mod tests { sequence_number: 0, last_data: false, }, - target_hostname: None, + target_hostname: "booga.com".to_string(), target_port: 0, protocol: ProxyProtocol::HTTP, originator_public_key: originator_public_key.clone(), diff --git a/node/src/proxy_client/stream_establisher.rs b/node/src/proxy_client/stream_establisher.rs index 602b3c0c1..a07e4d730 100644 --- a/node/src/proxy_client/stream_establisher.rs +++ b/node/src/proxy_client/stream_establisher.rs @@ -160,6 +160,8 @@ mod tests { fn spawn_stream_reader_handles_data() { let (proxy_client, proxy_client_awaiter, proxy_client_recording_arc) = make_recorder(); let (sub_tx, sub_rx) = unbounded(); + let stream_key = StreamKey::make_meaningless_stream_key(); + let stream_key_inner = stream_key.clone(); thread::spawn(move || { let system = System::new("spawn_stream_reader_handles_data"); let peer_actors = peer_actors_builder().proxy_client(proxy_client).build(); @@ -193,13 +195,13 @@ mod tests { }; subject.spawn_stream_reader( &ClientRequestPayload_0v1 { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key: stream_key_inner, sequenced_packet: SequencedPacket { data: vec![], sequence_number: 0, last_data: false, }, - target_hostname: Some("blah".to_string()), + target_hostname: "blah".to_string(), target_port: 0, protocol: ProxyProtocol::HTTP, originator_public_key: subject.cryptde.public_key().clone(), @@ -227,7 +229,7 @@ mod tests { assert_eq!( ibsd, InboundServerData { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key, last_data: false, sequence_number: 0, source: SocketAddr::from_str("1.2.3.4:5678").unwrap(), diff --git a/node/src/proxy_client/stream_handler_pool.rs b/node/src/proxy_client/stream_handler_pool.rs index 7fc623617..8a42ca2ae 100644 --- a/node/src/proxy_client/stream_handler_pool.rs +++ b/node/src/proxy_client/stream_handler_pool.rs @@ -114,11 +114,7 @@ impl StreamHandlerPoolReal { let inner_arc_1 = inner_arc.clone(); let logger = Self::make_logger_copy(&inner_arc); let data_len = payload.sequenced_packet.data.len(); - let hostname = payload - .target_hostname - .as_ref() - .unwrap_or(&"".to_string()) - .to_string(); + let hostname = payload.target_hostname.clone(); let target_port = payload.target_port; match Self::find_stream_with_key(&stream_key, &inner_arc) { Some(sender_wrapper) => { @@ -315,27 +311,14 @@ impl StreamHandlerPoolReal { "No stream to {:?} exists; resolving host", &payload.target_hostname ); - match payload.target_hostname { - Some(ref target_hostname) => match Self::parse_ip(target_hostname) { - Ok(socket_addr) => Self::handle_ip( - payload.clone(), - socket_addr, - inner_arc, - target_hostname.to_string(), - ), - Err(_) => Self::lookup_dns(inner_arc, target_hostname.to_string(), payload.clone()), - }, - None => { - error!( - logger, - "Cannot open new stream with key {:?}: no hostname supplied", - payload.stream_key - ); - Box::new(err::< - Box + 'static>, - String, - >("No hostname provided".to_string())) - } + match Self::parse_ip(&payload.target_hostname) { + Ok(socket_addr) => Self::handle_ip( + payload.clone(), + socket_addr, + inner_arc, + payload.target_hostname.clone(), + ), + Err(_) => Self::lookup_dns(inner_arc, payload.target_hostname.clone(), payload.clone()), } } @@ -732,7 +715,7 @@ mod tests { let payload = ClientRequestPayload_0v1 { stream_key, sequenced_packet: SequencedPacket::new(b"booga".to_vec(), 0, false), - target_hostname: Some("www.example.com".to_string()), + target_hostname: "www.example.com".to_string(), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: cryptde.public_key().clone(), @@ -765,7 +748,7 @@ mod tests { sequence_number: 0, last_data: false, }, - target_hostname: None, + target_hostname: "booga.com".to_string(), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(&b"men's souls"[..]), @@ -820,18 +803,20 @@ mod tests { init_test_logging(); let test_name = "write_failure_for_nonexistent_stream_generates_termination_message"; let cryptde = CRYPTDE_PAIR.main.as_ref(); + let stream_key = StreamKey::make_meaningless_stream_key(); + let stream_key_inner = stream_key.clone(); let (proxy_client, proxy_client_awaiter, proxy_client_recording_arc) = make_recorder(); let originator_key = PublicKey::new(&b"men's souls"[..]); let (reader_shutdown_tx, reader_shutdown_rx) = unbounded(); thread::spawn(move || { let client_request_payload = ClientRequestPayload_0v1 { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key: stream_key_inner, sequenced_packet: SequencedPacket { data: b"These are the times".to_vec(), sequence_number: 0, last_data: false, }, - target_hostname: Some(String::from("that.try")), + target_hostname: String::from("that.try"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: originator_key, @@ -879,7 +864,7 @@ mod tests { assert_eq!( proxy_client_recording.get_record::(0), &InboundServerData { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key: stream_key.clone(), last_data: true, sequence_number: 0, source: SocketAddr::from_str("2.3.4.5:80").unwrap(), @@ -888,7 +873,8 @@ mod tests { ); TestLogHandler::new().exists_log_containing(&format!( "DEBUG: {test_name}: A shutdown signal was sent to the StreamReader \ - for stream key AAAAAAAAAAAAAAAAAAAAAAAAAAA." + for stream key {}.", + stream_key )); } @@ -897,17 +883,19 @@ mod tests { let cryptde = CRYPTDE_PAIR.main.as_ref(); let write_parameters = Arc::new(Mutex::new(vec![])); let expected_write_parameters = write_parameters.clone(); + let stream_key = StreamKey::make_meaningless_stream_key(); + let stream_key_inner = stream_key.clone(); let (proxy_client, proxy_client_awaiter, proxy_client_recording_arc) = make_recorder(); thread::spawn(move || { let peer_actors = peer_actors_builder().proxy_client(proxy_client).build(); let client_request_payload = ClientRequestPayload_0v1 { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key: stream_key_inner, sequenced_packet: SequencedPacket { data: b"These are the times".to_vec(), sequence_number: 0, last_data: false, }, - target_hostname: Some(String::from("3.4.5.6:80")), + target_hostname: String::from("3.4.5.6:80"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(&b"men's souls"[..]), @@ -980,7 +968,7 @@ mod tests { assert_eq!( proxy_client_recording.get_record::(0), &InboundServerData { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key, last_data: false, sequence_number: 0, source: SocketAddr::from_str("3.4.5.6:80").unwrap(), @@ -1003,7 +991,7 @@ mod tests { sequence_number: 0, last_data: true, }, - target_hostname: Some(String::from("3.4.5.6:80")), + target_hostname: String::from("3.4.5.6:80"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(&b"brutal death"[..]), @@ -1064,7 +1052,7 @@ mod tests { sequence_number: 0, last_data: true, }, - target_hostname: Some(String::from("3.4.5.6:80")), + target_hostname: String::from("3.4.5.6:80"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(&b"brutal death"[..]), @@ -1116,17 +1104,19 @@ mod tests { let expected_lookup_ip_parameters = lookup_ip_parameters.clone(); let write_parameters = Arc::new(Mutex::new(vec![])); let expected_write_parameters = write_parameters.clone(); + let stream_key = StreamKey::make_meaningless_stream_key(); + let stream_key_inner = stream_key.clone(); let (proxy_client, proxy_client_awaiter, proxy_client_recording_arc) = make_recorder(); thread::spawn(move || { let peer_actors = peer_actors_builder().proxy_client(proxy_client).build(); let client_request_payload = ClientRequestPayload_0v1 { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key: stream_key_inner, sequenced_packet: SequencedPacket { data: b"These are the times".to_vec(), sequence_number: 0, last_data: false, }, - target_hostname: Some(String::from("3.4.5.6")), + target_hostname: String::from("3.4.5.6"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(&b"men's souls"[..]), @@ -1209,7 +1199,7 @@ mod tests { assert_eq!( proxy_client_recording.get_record::(0), &InboundServerData { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key, last_data: false, sequence_number: 0, source: SocketAddr::from_str("3.4.5.6:80").unwrap(), @@ -1218,71 +1208,6 @@ mod tests { ); } - #[test] - fn missing_hostname_for_nonexistent_stream_generates_log_and_termination_message() { - init_test_logging(); - let test_name = - "missing_hostname_for_nonexistent_stream_generates_log_and_termination_message"; - let cryptde = CRYPTDE_PAIR.main.as_ref(); - let (proxy_client, proxy_client_awaiter, proxy_client_recording_arc) = make_recorder(); - let originator_key = PublicKey::new(&b"men's souls"[..]); - let stream_key = StreamKey::make_meaningful_stream_key(test_name); - thread::spawn(move || { - let peer_actors = peer_actors_builder().proxy_client(proxy_client).build(); - let client_request_payload = ClientRequestPayload_0v1 { - stream_key: stream_key.clone(), - sequenced_packet: SequencedPacket { - data: b"These are the times".to_vec(), - sequence_number: 0, - last_data: false, - }, - target_hostname: None, - target_port: HTTP_PORT, - protocol: ProxyProtocol::HTTP, - originator_public_key: originator_key, - }; - let package = ExpiredCoresPackage::new( - SocketAddr::from_str("1.2.3.4:1234").unwrap(), - Some(make_wallet("consuming")), - make_meaningless_route(&CRYPTDE_PAIR), - client_request_payload.into(), - 0, - ); - let resolver = - ResolverWrapperMock::new().lookup_ip_failure(ResolveErrorKind::Io.into()); - let subject = StreamHandlerPoolReal::new( - Box::new(resolver), - cryptde, - peer_actors.accountant.report_exit_service_provided.clone(), - peer_actors.proxy_client_opt.unwrap().clone(), - 100, - 200, - ); - - run_process_package_in_actix(subject, package); - }); - - proxy_client_awaiter.await_message_count(1); - let proxy_client_recording = proxy_client_recording_arc.lock().unwrap(); - assert_eq!( - proxy_client_recording.get_record::(0), - &InboundServerData { - stream_key: stream_key.clone(), - last_data: true, - sequence_number: 0, - source: error_socket_addr(), - data: vec![], - } - ); - TestLogHandler::new().exists_log_containing( - format!( - "ERROR: ProxyClient: Cannot open new stream with key {:?}: no hostname supplied", - stream_key - ) - .as_str(), - ); - } - #[test] fn nonexistent_connection_springs_into_being_and_is_persisted_to_handle_transaction() { let cryptde = CRYPTDE_PAIR.main.as_ref(); @@ -1290,6 +1215,8 @@ mod tests { let expected_lookup_ip_parameters = lookup_ip_parameters.clone(); let write_parameters = Arc::new(Mutex::new(vec![])); let expected_write_parameters = write_parameters.clone(); + let stream_key = StreamKey::make_meaningless_stream_key(); + let stream_key_inner = stream_key.clone(); let (proxy_client, proxy_client_awaiter, proxy_client_recording_arc) = make_recorder(); let (accountant, accountant_awaiter, accountant_recording_arc) = make_recorder(); let before = SystemTime::now(); @@ -1299,13 +1226,13 @@ mod tests { .accountant(accountant) .build(); let client_request_payload = ClientRequestPayload_0v1 { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key: stream_key_inner, sequenced_packet: SequencedPacket { data: b"These are the times".to_vec(), sequence_number: 0, last_data: false, }, - target_hostname: Some(String::from("that.try")), + target_hostname: String::from("that.try"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(&b"men's souls"[..]), @@ -1390,7 +1317,7 @@ mod tests { assert_eq!( proxy_client_recording.get_record::(0), &InboundServerData { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key, last_data: false, sequence_number: 0, source: SocketAddr::from_str("3.4.5.6:80").unwrap(), @@ -1418,7 +1345,7 @@ mod tests { sequence_number: 0, last_data: false, }, - target_hostname: Some(String::from("that.try")), + target_hostname: String::from("that.try"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: originator_key, @@ -1540,7 +1467,7 @@ mod tests { sequence_number: 0, last_data: false, }, - target_hostname: Some(String::from("blockedwebsite.com")), + target_hostname: String::from("blockedwebsite.com"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: originator_key, @@ -1637,7 +1564,7 @@ mod tests { let client_request_payload = ClientRequestPayload_0v1 { stream_key, sequenced_packet: sequenced_packet.clone(), - target_hostname: Some(String::from("that.try")), + target_hostname: String::from("that.try"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(&b"men's souls"[..]), @@ -1750,7 +1677,7 @@ mod tests { sequence_number: 0, last_data: true, }, - target_hostname: Some(String::from("that.try")), + target_hostname: String::from("that.try"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: originator_key, @@ -1812,7 +1739,7 @@ mod tests { let client_request_payload = ClientRequestPayload_0v1 { stream_key: stream_key.clone(), sequenced_packet: sequenced_packet.clone(), - target_hostname: Some(String::from("that.try")), + target_hostname: String::from("that.try"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(&b"men's souls"[..]), @@ -1883,7 +1810,7 @@ mod tests { sequence_number: 0, last_data: false, }, - target_hostname: None, + target_hostname: "booga.com".to_string(), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(&b"booga"[..]), diff --git a/node/src/proxy_client/stream_reader.rs b/node/src/proxy_client/stream_reader.rs index 992b58dbf..bf39de6e3 100644 --- a/node/src/proxy_client/stream_reader.rs +++ b/node/src/proxy_client/stream_reader.rs @@ -179,9 +179,10 @@ mod tests { }); let proxy_client_sub = rx.recv().unwrap(); + let stream_key = StreamKey::make_meaningless_stream_key(); let (stream_killer, stream_killer_params) = unbounded(); let mut subject = StreamReader { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key: stream_key.clone(), proxy_client_sub, stream, stream_killer, @@ -198,7 +199,7 @@ mod tests { assert_eq!( proxy_client_recording.get_record::(0), &InboundServerData { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key: stream_key.clone(), last_data: false, sequence_number: 0, source: SocketAddr::from_str("8.7.4.3:50").unwrap(), @@ -208,7 +209,7 @@ mod tests { assert_eq!( proxy_client_recording.get_record::(1), &InboundServerData { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key: stream_key.clone(), last_data: false, sequence_number: 1, source: SocketAddr::from_str("8.7.4.3:50").unwrap(), @@ -218,7 +219,7 @@ mod tests { assert_eq!( proxy_client_recording.get_record::(2), &InboundServerData { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key: stream_key.clone(), last_data: false, sequence_number: 2, source: SocketAddr::from_str("8.7.4.3:50").unwrap(), @@ -226,10 +227,7 @@ mod tests { }, ); let stream_killer_parameters = stream_killer_params.try_recv().unwrap(); - assert_eq!( - stream_killer_parameters, - (StreamKey::make_meaningless_stream_key(), 3) - ); + assert_eq!(stream_killer_parameters, (stream_key, 3)); } #[test] @@ -266,6 +264,7 @@ mod tests { let (stream_killer, stream_killer_params) = unbounded(); let peer_addr = SocketAddr::from_str("5.7.9.0:95").unwrap(); let mut subject = make_subject(); + let stream_key = subject.stream_key.clone(); subject.proxy_client_sub = proxy_client_sub; subject.stream = Box::new(stream); subject.stream_killer = stream_killer; @@ -279,7 +278,7 @@ mod tests { assert_eq!( proxy_client_recording.get_record::(0), &InboundServerData { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key: stream_key.clone(), last_data: false, sequence_number: 0, source: peer_addr, @@ -289,7 +288,7 @@ mod tests { assert_eq!( proxy_client_recording.get_record::(1), &InboundServerData { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key: stream_key.clone(), last_data: false, sequence_number: 1, source: peer_addr, @@ -299,7 +298,7 @@ mod tests { assert_eq!( proxy_client_recording.get_record::(2), &InboundServerData { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key: stream_key.clone(), last_data: false, sequence_number: 2, source: peer_addr, @@ -310,10 +309,7 @@ mod tests { let kill_stream_msg = stream_killer_params .try_recv() .expect("stream was not killed"); - assert_eq!( - kill_stream_msg, - (StreamKey::make_meaningless_stream_key(), 3) - ); + assert_eq!(kill_stream_msg, (stream_key, 3)); assert!(stream_killer_params.try_recv().is_err()); } diff --git a/node/src/proxy_server/client_request_payload_factory.rs b/node/src/proxy_server/client_request_payload_factory.rs index 75a484a37..31e7182d4 100644 --- a/node/src/proxy_server/client_request_payload_factory.rs +++ b/node/src/proxy_server/client_request_payload_factory.rs @@ -3,6 +3,7 @@ use crate::proxy_server::protocol_pack::from_ibcd; use crate::sub_lib::cryptde::CryptDE; use crate::sub_lib::cryptde::PlainData; use crate::sub_lib::dispatcher::InboundClientData; +use crate::sub_lib::host::Host; use crate::sub_lib::proxy_server::ClientRequestPayload_0v1; use crate::sub_lib::sequence_buffer::SequencedPacket; use crate::sub_lib::stream_key::StreamKey; @@ -13,6 +14,7 @@ pub trait ClientRequestPayloadFactory { &self, ibcd: &InboundClientData, stream_key: StreamKey, + host_opt: Option, cryptde: &dyn CryptDE, logger: &Logger, ) -> Option; @@ -26,11 +28,43 @@ impl ClientRequestPayloadFactory for ClientRequestPayloadFactoryReal { &self, ibcd: &InboundClientData, stream_key: StreamKey, + host_from_history_opt: Option, cryptde: &dyn CryptDE, logger: &Logger, ) -> Option { let protocol_pack = from_ibcd(ibcd).map_err(|e| error!(logger, "{}", e)).ok()?; - let sequence_number = match ibcd.sequence_number { + let host_from_request_result_closure = Box::new(|| { + let data = PlainData::new(&ibcd.data); + match protocol_pack.find_host(&data) { + Some(host) => Ok(host), + // So far we've only looked in the client packet; but this message will evaporate + // unless there's no host information in host_opt (from ProxyServer's StreamInfo) either. + None => Err(format!( + "No hostname information found in either client packet ({}) or ProxyServer for protocol {:?}, with StreamKey {}", + protocol_pack.describe_packet(&data), + protocol_pack.proxy_protocol(), + stream_key + )), + } + }); + let target_host = match (host_from_request_result_closure(), host_from_history_opt) { + (Ok(host), _) => host, + (Err(_), Some(host)) => host, + (Err(e), None) => { + if ibcd.last_data && ibcd.data.is_empty() { + warning!( + logger, + "Client opened {:?} connection and immediately closed it without sending any data, with StreamKey {}", + protocol_pack.proxy_protocol(), + stream_key + ); + } else { + error!(logger, "{}", e); + } + return None; + } + }; + let sequence_number = match ibcd.sequence_number_opt { Some(sequence_number) => sequence_number, None => { error!( @@ -41,12 +75,6 @@ impl ClientRequestPayloadFactory for ClientRequestPayloadFactoryReal { return None; } }; - let data = PlainData::new(&ibcd.data); - let target_host = protocol_pack.find_host(&data); - let (target_hostname_opt, target_port) = match target_host { - Some(host) => (Some(host.name), host.port), - None => (None, protocol_pack.standard_port()), - }; Some(ClientRequestPayload_0v1 { stream_key, sequenced_packet: SequencedPacket { @@ -54,8 +82,8 @@ impl ClientRequestPayloadFactory for ClientRequestPayloadFactoryReal { sequence_number, last_data: ibcd.last_data, }, - target_hostname: target_hostname_opt, - target_port, + target_hostname: target_host.name, + target_port: target_host.port, protocol: protocol_pack.proxy_protocol(), originator_public_key: cryptde.public_key().clone(), }) @@ -74,7 +102,7 @@ mod tests { use crate::bootstrapper::CryptDEPair; use crate::sub_lib::proxy_server::ProxyProtocol; use lazy_static::lazy_static; - use masq_lib::constants::HTTP_PORT; + use masq_lib::constants::{HTTP_PORT, TLS_PORT}; use masq_lib::test_utils::logging::init_test_logging; use masq_lib::test_utils::logging::TestLogHandler; use std::net::SocketAddr; @@ -85,14 +113,127 @@ mod tests { static ref CRYPTDE_PAIR: CryptDEPair = CryptDEPair::null(); } + #[test] + fn ibcd_hostname_overrides_supplied_hostname() { + let data = PlainData::new(&b"GET http://borkoed.com:1234/fleebs.html HTTP/1.1\r\n\r\n"[..]); + let ibcd = InboundClientData { + timestamp: SystemTime::now(), + client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(1), + last_data: false, + is_clandestine: false, + data: data.clone().into(), + }; + let cryptde = CRYPTDE_PAIR.main.dup(); + let stream_key = StreamKey::make_meaningless_stream_key(); + let logger = Logger::new("ibcd_hostname_overrides_supplied_hostname"); + let subject = Box::new(ClientRequestPayloadFactoryReal::new()); + + let result = subject + .make( + &ibcd, + stream_key, + Some(Host::new("ignored.com", 4321)), + cryptde.as_ref(), + &logger, + ) + .unwrap(); + + assert_eq!(result.target_hostname, String::from("borkoed.com")); + assert_eq!(result.target_port, 1234); + } + + #[test] + fn uses_supplied_host_if_ibcd_does_not_have_one() { + let test_name = "uses_supplied_hostname_if_ibcd_does_not_have_one"; + let data = PlainData::new(&[0x01, 0x02, 0x03]); // No host can be extracted here + let ibcd = InboundClientData { + timestamp: SystemTime::now(), + client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(1), + last_data: false, + is_clandestine: false, + data: data.into(), + }; + let cryptde = CRYPTDE_PAIR.main.dup(); + let stream_key = StreamKey::make_meaningful_stream_key(test_name); + let logger = Logger::new(test_name); + let subject = Box::new(ClientRequestPayloadFactoryReal::new()); + let supplied_host = Host::new("supplied.com", 4321); + + let result = subject + .make( + &ibcd, + stream_key, + Some(supplied_host.clone()), + cryptde.as_ref(), + &logger, + ) + .unwrap(); + + assert_eq!(result.target_hostname, supplied_host.name); + } + + #[test] + fn logs_error_and_returns_none_if_no_ibcd_host_and_no_supplied_host() { + init_test_logging(); + let test_name = "logs_error_and_returns_none_if_no_ibcd_hostname_and_no_supplied_hostname"; + let data = PlainData::new(&[0x01, 0x02, 0x03]); // no host can be extracted here + let ibcd = InboundClientData { + timestamp: SystemTime::now(), + client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(1), + last_data: false, + is_clandestine: false, + data: data.into(), + }; + let cryptde = CRYPTDE_PAIR.main.dup(); + let stream_key = StreamKey::make_meaningful_stream_key(test_name); + let logger = Logger::new(test_name); + let subject = Box::new(ClientRequestPayloadFactoryReal::new()); + + let result = subject.make(&ibcd, stream_key, None, cryptde.as_ref(), &logger); + + assert_eq!(result, None); + TestLogHandler::new().exists_log_containing(&format!("ERROR: {test_name}: No hostname information found in either client packet (Malformed HTTP request: '') or ProxyServer for protocol HTTP")); + } + + #[test] + fn logs_different_error_and_returns_none_if_connection_is_opened_and_immediately_closed() { + init_test_logging(); + let test_name = + "logs_different_error_and_returns_none_if_connection_is_opened_and_immediately_closed"; + let ibcd = InboundClientData { + timestamp: SystemTime::now(), + client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(1), + last_data: true, + is_clandestine: false, + data: vec![], + }; + let cryptde = CRYPTDE_PAIR.main.dup(); + let stream_key = StreamKey::make_meaningful_stream_key(test_name); + let logger = Logger::new(test_name); + let subject = Box::new(ClientRequestPayloadFactoryReal::new()); + + let result = subject.make(&ibcd, stream_key, None, cryptde.as_ref(), &logger); + + assert_eq!(result, None); + TestLogHandler::new().exists_log_containing(&format!("WARN: {test_name}: Client opened HTTP connection and immediately closed it without sending any data")); + } + #[test] fn handles_http_with_a_port() { let data = PlainData::new(&b"GET http://borkoed.com:2345/fleebs.html HTTP/1.1\r\n\r\n"[..]); let ibcd = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(1), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(1), last_data: false, is_clandestine: false, data: data.clone().into(), @@ -102,7 +243,7 @@ mod tests { let logger = Logger::new("test"); let subject = Box::new(ClientRequestPayloadFactoryReal::new()); - let result = subject.make(&ibcd, stream_key, cryptde, &logger); + let result = subject.make(&ibcd, stream_key, None, cryptde, &logger); assert_eq!( result, @@ -113,7 +254,7 @@ mod tests { sequence_number: 1, last_data: false }, - target_hostname: Some(String::from("borkoed.com")), + target_hostname: String::from("borkoed.com"), target_port: 2345, protocol: ProxyProtocol::HTTP, originator_public_key: cryptde.public_key().clone(), @@ -128,8 +269,8 @@ mod tests { let ibcd = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(1), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(1), last_data: false, is_clandestine: false, data: data.clone().into(), @@ -139,7 +280,7 @@ mod tests { let stream_key = StreamKey::make_meaningful_stream_key(test_name); let subject = Box::new(ClientRequestPayloadFactoryReal::new()); - let result = subject.make(&ibcd, stream_key, cryptde, &logger); + let result = subject.make(&ibcd, stream_key, None, cryptde, &logger); assert_eq!( result, @@ -150,7 +291,7 @@ mod tests { sequence_number: 1, last_data: false }, - target_hostname: Some(String::from("borkoed.com")), + target_hostname: String::from("borkoed.com"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: cryptde.public_key().clone(), @@ -183,8 +324,8 @@ mod tests { let ibcd = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - sequence_number: Some(0), - reception_port: Some(443), + sequence_number_opt: Some(0), + reception_port_opt: Some(443), last_data: false, is_clandestine: false, data: data.clone().into(), @@ -194,7 +335,7 @@ mod tests { let logger = Logger::new("test"); let subject = Box::new(ClientRequestPayloadFactoryReal::new()); - let result = subject.make(&ibcd, stream_key, cryptde, &logger); + let result = subject.make(&ibcd, stream_key, None, cryptde, &logger); assert_eq!( result, @@ -205,8 +346,8 @@ mod tests { sequence_number: 0, last_data: false }, - target_hostname: Some(String::from("server.com")), - target_port: 443, + target_hostname: String::from("server.com"), + target_port: TLS_PORT, protocol: ProxyProtocol::TLS, originator_public_key: cryptde.public_key().clone(), }) @@ -215,6 +356,7 @@ mod tests { #[test] fn handles_tls_without_hostname() { + init_test_logging(); let test_name = "handles_tls_without_hostname"; let data = PlainData::new(&[ 0x16, // content_type: Handshake @@ -233,10 +375,10 @@ mod tests { let ibcd = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: Some(443), + reception_port_opt: Some(443), last_data: true, is_clandestine: false, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: data.clone().into(), }; let cryptde = CRYPTDE_PAIR.main.as_ref(); @@ -244,23 +386,10 @@ mod tests { let stream_key = StreamKey::make_meaningful_stream_key(test_name); let subject = Box::new(ClientRequestPayloadFactoryReal::new()); - let result = subject.make(&ibcd, stream_key, cryptde, &logger); + let result = subject.make(&ibcd, stream_key, None, cryptde, &logger); - assert_eq!( - result, - Some(ClientRequestPayload_0v1 { - stream_key, - sequenced_packet: SequencedPacket { - data: data.into(), - sequence_number: 0, - last_data: true - }, - target_hostname: None, - target_port: 443, - protocol: ProxyProtocol::TLS, - originator_public_key: cryptde.public_key().clone(), - }) - ); + assert_eq!(result, None); + TestLogHandler::new().exists_log_containing(&format!("ERROR: {test_name}: No hostname information found in either client packet (ClientHello with no SNI extension) or ProxyServer for protocol TLS")); } #[test] @@ -270,8 +399,8 @@ mod tests { let ibcd = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - sequence_number: Some(0), - reception_port: None, + sequence_number_opt: Some(0), + reception_port_opt: None, last_data: false, is_clandestine: false, data: vec![0x10, 0x11, 0x12], @@ -281,7 +410,7 @@ mod tests { let stream_key = StreamKey::make_meaningful_stream_key(test_name); let subject = Box::new(ClientRequestPayloadFactoryReal::new()); - let result = subject.make(&ibcd, stream_key, cryptde, &logger); + let result = subject.make(&ibcd, stream_key, None, cryptde, &logger); assert_eq!(result, None); TestLogHandler::new().exists_log_containing( @@ -296,8 +425,8 @@ mod tests { let ibcd = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), - reception_port: Some(1234), - sequence_number: Some(0), + reception_port_opt: Some(1234), + sequence_number_opt: Some(0), last_data: false, is_clandestine: true, data: vec![0x10, 0x11, 0x12], @@ -307,7 +436,7 @@ mod tests { let stream_key = StreamKey::make_meaningful_stream_key(test_name); let subject = Box::new(ClientRequestPayloadFactoryReal::new()); - let result = subject.make(&ibcd, stream_key, cryptde, &logger); + let result = subject.make(&ibcd, stream_key, None, cryptde, &logger); assert_eq!(result, None); TestLogHandler::new().exists_log_containing(&format!("ERROR: {test_name}: No protocol associated with origin port 1234 for 3-byte non-clandestine packet: [16, 17, 18]")); @@ -315,13 +444,14 @@ mod tests { #[test] fn use_sequence_from_inbound_client_data_in_client_request_payload() { + let data = PlainData::new(&b"GET http://borkoed.com/fleebs.html HTTP/1.1\r\n\r\n"[..]); let ibcd = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:80").unwrap(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(1), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(1), last_data: false, - data: vec![0x10, 0x11, 0x12], + data: data.into(), is_clandestine: false, }; let cryptde = CRYPTDE_PAIR.main.as_ref(); @@ -332,6 +462,7 @@ mod tests { .make( &ibcd, StreamKey::make_meaningless_stream_key(), + None, cryptde, &logger, ) @@ -344,25 +475,26 @@ mod tests { fn makes_no_payload_if_sequence_number_is_unknown() { init_test_logging(); let test_name = "makes_no_payload_if_sequence_number_is_unknown"; + let data = PlainData::new(&b"GET http://borkoed.com/fleebs.html HTTP/1.1\r\n\r\n"[..]); let ibcd = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:80").unwrap(), - reception_port: Some(HTTP_PORT), + reception_port_opt: Some(HTTP_PORT), last_data: false, is_clandestine: false, - sequence_number: None, - data: vec![1, 3, 5, 7], + sequence_number_opt: None, + data: data.into(), }; let cryptde = CRYPTDE_PAIR.main.as_ref(); let logger = Logger::new(test_name); let stream_key = StreamKey::make_meaningful_stream_key(test_name); let subject = Box::new(ClientRequestPayloadFactoryReal::new()); - let result = subject.make(&ibcd, stream_key, cryptde, &logger); + let result = subject.make(&ibcd, stream_key, None, cryptde, &logger); assert_eq!(result, None); TestLogHandler::new().exists_log_containing(&format!( - "ERROR: {test_name}: internal error: got IBCD with no sequence number and 4 bytes" + "ERROR: {test_name}: internal error: got IBCD with no sequence number and 47 bytes" )); } } diff --git a/node/src/proxy_server/http_protocol_pack.rs b/node/src/proxy_server/http_protocol_pack.rs index b611f2be3..ef8940c6d 100644 --- a/node/src/proxy_server/http_protocol_pack.rs +++ b/node/src/proxy_server/http_protocol_pack.rs @@ -1,7 +1,8 @@ // Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. -use crate::proxy_server::protocol_pack::{Host, ProtocolPack, ServerImpersonator}; +use crate::proxy_server::protocol_pack::{ProtocolPack, ServerImpersonator}; use crate::proxy_server::server_impersonator_http::ServerImpersonatorHttp; use crate::sub_lib::cryptde::PlainData; +use crate::sub_lib::host::Host; use crate::sub_lib::proxy_server::ProxyProtocol; use lazy_static::lazy_static; use masq_lib::constants::HTTP_PORT; @@ -12,6 +13,7 @@ lazy_static! { static ref HOST_PATTERN: Regex = Regex::new(r"^(?:https?://)?([^\s/]+)").expect("bad regex"); } +#[derive(Clone, Copy)] pub struct HttpProtocolPack {} impl ProtocolPack for HttpProtocolPack { @@ -33,6 +35,14 @@ impl ProtocolPack for HttpProtocolPack { fn server_impersonator(&self) -> Box { Box::new(ServerImpersonatorHttp {}) } + + fn describe_packet(&self, data: &PlainData) -> String { + if data.as_slice().starts_with(b"HTTP/") { + self.describe_response(data) + } else { + self.describe_request(data) + } + } } impl HttpProtocolPack { @@ -100,6 +110,82 @@ impl HttpProtocolPack { Ok(port) => Ok(port), } } + + fn describe_request(&self, data: &PlainData) -> String { + let first_line_end = data + .as_slice() + .iter() + .position(|&b| b == b'\r') + .unwrap_or(data.len()); + let first_line = &data.as_slice()[0..first_line_end]; + let mut parts = first_line.split(|&b| b == b' '); + if let (Some(method_bytes), Some(path_bytes), Some(http_version_bytes)) = + (parts.next(), parts.next(), parts.next()) + { + let method = Self::from_utf8(method_bytes); + let path = Self::from_utf8(path_bytes); + let http_version = Self::from_utf8(http_version_bytes); + if let Some(host) = self.find_host(data) { + return if path.starts_with('/') { + format!( + "{} {} request to {}{}", + http_version, method, host.name, path + ) + } else { + format!("{} {} request to {}", http_version, method, path) + }; + } else { + return format!( + "{} {} request to unknown host: {}", + http_version, method, path + ); + } + } + format!( + "Malformed HTTP request: {}", + Self::truncate_data_as_string(data, 50) + ) + } + + fn describe_response(&self, data: &PlainData) -> String { + let first_line_end = data + .as_slice() + .iter() + .position(|&b| b == b'\r') + .unwrap_or(data.as_slice().len()); + let first_line = &data.as_slice()[0..first_line_end]; + let mut parts = first_line.split(|&b| b == b' '); + if let (Some(http_version_bytes), Some(status_code_bytes), Some(status_text_bytes)) = + (parts.next(), parts.next(), parts.next()) + { + let http_with_version = Self::from_utf8(http_version_bytes); + let status_code = Self::from_utf8(status_code_bytes); + let status_text = Self::from_utf8(status_text_bytes); + return format!( + "{} response with status {} {}", + http_with_version, status_code, status_text + ); + } + format!( + "Malformed HTTP response: {}", + Self::truncate_data_as_string(data, 50) + ) + } + + fn truncate_data_as_string(data: &PlainData, truncate_at: usize) -> String { + if data.as_slice().len() > truncate_at { + format!( + "'{}'...", + String::from_utf8_lossy(&data.as_slice()[0..truncate_at]) + ) + } else { + format!("'{}'", String::from_utf8_lossy(data.as_slice())) + } + } + + fn from_utf8(bytes: &[u8]) -> String { + String::from_utf8_lossy(bytes).to_string() + } } #[cfg(test)] @@ -336,4 +422,83 @@ mod tests { let data = b"CONNECTX"; assert!(!HttpProtocolPack::is_connect(data)); } + + #[test] + fn describe_packet_works_on_get_request_with_header_host() { + let data = + PlainData::new(b"GET /index.html?item=booga HTTP/1.1\r\nHost: www.example.com\r\n\r\n"); + let subject = HttpProtocolPack {}; + + let result = subject.describe_packet(&data); + + assert_eq!( + result, + "HTTP/1.1 GET request to www.example.com/index.html?item=booga" + ); + } + + #[test] + fn describe_packet_works_on_post_request_with_url_host() { + let data = PlainData::new( + b"POST www.example.com/person/1234 HTTP/1.1\r\nContent-Length: 2\r\n\r\n{}", + ); + let subject = HttpProtocolPack {}; + + let result = subject.describe_packet(&data); + + assert_eq!( + result, + "HTTP/1.1 POST request to www.example.com/person/1234" + ); + } + + #[test] + fn describe_packet_works_on_unexpected_request_with_unspecified_host() { + let data = PlainData::new(b"BOOGA /person/1234 HTTP/1.1\r\nContent-Length: 2\r\n\r\n{}"); + let subject = HttpProtocolPack {}; + + let result = subject.describe_packet(&data); + + assert_eq!( + result, + "HTTP/1.1 BOOGA request to unknown host: /person/1234" + ); + } + + #[test] + fn describe_packet_works_on_malformed_request() { + let data = PlainData::new(b"Fourscore_and_seven_years_ago_our_fathers_brought_forth_on_this_continent_a_new_nation,_conceived_in_liberty_and_dedicated_to_the_proposition_that_all_men_are_created_equal."); + let subject = HttpProtocolPack {}; + + let result = subject.describe_packet(&data); + + assert_eq!( + result, + "Malformed HTTP request: 'Fourscore_and_seven_years_ago_our_fathers_brought_'..." + ); + } + + #[test] + fn describe_packet_works_on_200_response() { + let data = + PlainData::new(b"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n"); + let subject = HttpProtocolPack {}; + + let result = subject.describe_packet(&data); + + assert_eq!(result, "HTTP/1.1 response with status 200 OK"); + } + + #[test] + fn describe_packet_works_on_malformed_response() { + let data = PlainData::new(b"HTTP/Fourscore_and_seven_years_ago_our_fathers_brought_forth_on_this_continent_a_new_nation,_conceived_in_liberty_and_dedicated_to_the_proposition_that_all_men_are_created_equal."); + let subject = HttpProtocolPack {}; + + let result = subject.describe_packet(&data); + + assert_eq!( + result, + "Malformed HTTP response: 'HTTP/Fourscore_and_seven_years_ago_our_fathers_bro'..." + ); + } } diff --git a/node/src/proxy_server/mod.rs b/node/src/proxy_server/mod.rs index ec9166bc5..cfd7c54f0 100644 --- a/node/src/proxy_server/mod.rs +++ b/node/src/proxy_server/mod.rs @@ -24,6 +24,7 @@ use crate::sub_lib::cryptde::PublicKey; use crate::sub_lib::dispatcher::InboundClientData; use crate::sub_lib::dispatcher::{Endpoint, StreamShutdownMsg}; use crate::sub_lib::hopper::{ExpiredCoresPackage, IncipientCoresPackage}; +use crate::sub_lib::host::Host; use crate::sub_lib::neighborhood::RouteQueryResponse; use crate::sub_lib::neighborhood::{ExpectedService, UpdateNodeRecordMetadataMessage}; use crate::sub_lib::neighborhood::{ExpectedServices, RatePack}; @@ -31,14 +32,13 @@ use crate::sub_lib::neighborhood::{NRMetadataChange, RouteQueryMessage}; use crate::sub_lib::peer_actors::BindMessage; use crate::sub_lib::proxy_client::{ClientResponsePayload_0v1, DnsResolveFailure_0v1}; use crate::sub_lib::proxy_server::ProxyServerSubs; -use crate::sub_lib::proxy_server::{AddReturnRouteMessage, StreamKeyPurge}; +use crate::sub_lib::proxy_server::StreamKeyPurge; use crate::sub_lib::proxy_server::{ AddRouteResultMessage, ClientRequestPayload_0v1, ProxyProtocol, }; use crate::sub_lib::route::Route; use crate::sub_lib::stream_handler_pool::TransmitDataMsg; use crate::sub_lib::stream_key::StreamKey; -use crate::sub_lib::ttl_hashmap::TtlHashMap; use crate::sub_lib::utils::{handle_ui_crash_request, MessageScheduler, NODE_MAILBOX_CAPACITY}; use crate::sub_lib::wallet::Wallet; use actix::Context; @@ -51,17 +51,16 @@ use masq_lib::logger::Logger; use masq_lib::ui_gateway::NodeFromUiMessage; use masq_lib::utils::MutabilityConflictHelper; use regex::Regex; +use std::cell::Cell; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; -use std::rc::Rc; use std::str::FromStr; use std::time::{Duration, SystemTime}; use tokio::prelude::Future; pub const CRASH_KEY: &str = "PROXYSERVER"; -pub const RETURN_ROUTE_TTL: Duration = Duration::from_secs(120); - pub const STREAM_KEY_PURGE_DELAY: Duration = Duration::from_secs(30); +pub const DNS_FAILURE_RETRIES: usize = 3; struct ProxyServerOutSubs { dispatcher: Recipient, @@ -69,30 +68,35 @@ struct ProxyServerOutSubs { accountant: Recipient, route_source: Recipient, update_node_record_metadata: Recipient, - add_return_route: Recipient, stream_shutdown_sub: Recipient, route_result_sub: Recipient, schedule_stream_key_purge: Recipient>, } +#[derive(Clone, Debug)] +struct StreamInfo { + tunneled_host_opt: Option, + dns_failure_retry_opt: Option, + route_opt: Option, + protocol_opt: Option, + time_to_live_opt: Option, +} + pub struct ProxyServer { subs: Option, client_request_payload_factory: Box, stream_key_factory: Box, keys_and_addrs: BidiHashMap, - tunneled_hosts: HashMap, - dns_failure_retries: HashMap, - stream_key_routes: HashMap, - stream_key_ttl: HashMap, + stream_info: HashMap, is_decentralized: bool, consuming_wallet_balance: Option, cryptde_pair: CryptDEPair, crashable: bool, logger: Logger, - route_ids_to_return_routes: TtlHashMap, browser_proxy_sequence_offset: bool, inbound_client_data_helper_opt: Option>, stream_key_purge_delay: Duration, + next_return_route_id: Cell, is_running_in_integration_test: bool, } @@ -111,7 +115,6 @@ impl Handler for ProxyServer { accountant: msg.peer_actors.accountant.report_services_consumed, route_source: msg.peer_actors.neighborhood.route_query, update_node_record_metadata: msg.peer_actors.neighborhood.update_node_record_metadata, - add_return_route: msg.peer_actors.proxy_server.add_return_route, stream_shutdown_sub: msg.peer_actors.proxy_server.stream_shutdown_sub, route_result_sub: msg.peer_actors.proxy_server.route_result_sub, schedule_stream_key_purge: msg.peer_actors.proxy_server.schedule_stream_key_purge, @@ -128,64 +131,19 @@ impl Handler for ProxyServer { self.tls_connect(&msg); self.browser_proxy_sequence_offset = true; } else if let Err(e) = - self.help(|helper, proxy| helper.handle_normal_client_data(proxy, msg, false)) + // NOTE: I removed a 'false' parameter here for retire_stream_key because I think it was wrong. + self.help(|helper, proxy| helper.handle_normal_client_data(proxy, msg)) { error!(self.logger, "{}", e) } } } -impl Handler for ProxyServer { - type Result = (); - - fn handle(&mut self, msg: AddReturnRouteMessage, _ctx: &mut Self::Context) -> Self::Result { - self.route_ids_to_return_routes - .insert(msg.return_route_id, msg); - } -} - -impl AddReturnRouteMessage { - pub fn find_exit_node_key(&self) -> Option<&PublicKey> { - self.expected_services - .iter() - .find_map(|service| match service { - ExpectedService::Exit(public_key, _, _) => Some(public_key), - _ => None, - }) - } - - pub fn is_zero_hop(&self) -> bool { - self.expected_services == vec![ExpectedService::Nothing, ExpectedService::Nothing] - } -} - impl Handler for ProxyServer { type Result = (); fn handle(&mut self, msg: AddRouteResultMessage, _ctx: &mut Self::Context) -> Self::Result { - let dns_failure = self - .dns_failure_retries - .get(&msg.stream_key) - .unwrap_or_else(|| { - panic!("AddRouteResultMessage Handler: stream key: {} not found within dns_failure_retries", msg.stream_key); - }); - - match msg.result { - Ok(route_query_response) => { - debug!( - self.logger, - "Found a new route for hostname: {:?} - stream key: {} retries left: {}", - dns_failure.unsuccessful_request.target_hostname, - msg.stream_key, - dns_failure.retries_left - ); - self.stream_key_routes - .insert(msg.stream_key, route_query_response); - } - Err(e) => { - warning!(self.logger, "No route found for hostname: {:?} - stream key {} - retries left: {} - AddRouteResultMessage Error: {}",dns_failure.unsuccessful_request.target_hostname, msg.stream_key, dns_failure.retries_left, e); - } - } + self.handle_add_route_result_message(msg) } } @@ -256,24 +214,22 @@ impl ProxyServer { crashable: bool, is_running_in_integration_test: bool, ) -> ProxyServer { + let ps_logger = Logger::new("ProxyServer"); ProxyServer { subs: None, client_request_payload_factory: Box::new(ClientRequestPayloadFactoryReal::new()), stream_key_factory: Box::new(StreamKeyFactoryReal {}), keys_and_addrs: BidiHashMap::new(), - tunneled_hosts: HashMap::new(), - dns_failure_retries: HashMap::new(), - stream_key_routes: HashMap::new(), - stream_key_ttl: HashMap::new(), + stream_info: HashMap::new(), is_decentralized, consuming_wallet_balance, cryptde_pair, crashable, - logger: Logger::new("ProxyServer"), - route_ids_to_return_routes: TtlHashMap::new(RETURN_ROUTE_TTL), + logger: ps_logger, browser_proxy_sequence_offset: false, inbound_client_data_helper_opt: Some(Box::new(IBCDHelperReal::new())), stream_key_purge_delay: STREAM_KEY_PURGE_DELAY, + next_return_route_id: Cell::new(1), is_running_in_integration_test, } } @@ -284,7 +240,6 @@ impl ProxyServer { from_dispatcher: recipient!(addr, InboundClientData), from_hopper: recipient!(addr, ExpiredCoresPackage), dns_failure_from_hopper: recipient!(addr, ExpiredCoresPackage), - add_return_route: recipient!(addr, AddReturnRouteMessage), stream_shutdown_sub: recipient!(addr, StreamShutdownMsg), node_from_ui: recipient!(addr, NodeFromUiMessage), route_result_sub: recipient!(addr, AddRouteResultMessage), @@ -292,24 +247,46 @@ impl ProxyServer { } } + fn stream_info(&self, stream_key: &StreamKey) -> Option<&StreamInfo> { + match self.stream_info.get(stream_key) { + None => { + error!( + self.logger, + "Stream key {} not found in stream_info", stream_key + ); + None + } + Some(info) => Some(info), + } + } + + fn stream_info_mut(&mut self, stream_key: &StreamKey) -> Option<&mut StreamInfo> { + match self.stream_info.get_mut(stream_key) { + None => { + error!( + self.logger, + "Stream key {} not found in stream_info", stream_key + ); + None + } + Some(info) => Some(info), + } + } + fn remove_dns_failure_retry( - &mut self, + stream_info: &mut StreamInfo, stream_key: &StreamKey, ) -> Result { - match self.dns_failure_retries.remove(stream_key) { + match stream_info.dns_failure_retry_opt.take() { None => Err(format!( - "No entry found inside dns_failure_retries hashmap for the stream_key: {:?}", + "No DNSFailureRetry entry found for the stream_key: {:?}", stream_key )), Some(retry) => Ok(retry), } } - fn retry_dns_resolution( - &mut self, - retry: DNSFailureRetry, - client_addr: SocketAddr, - ) -> DNSFailureRetry { + fn retry_dns_resolution(&mut self, retry: &DNSFailureRetry, client_addr: SocketAddr) { let args = TransmitToHopperArgs::new( self, retry.unsuccessful_request.clone(), @@ -317,7 +294,6 @@ impl ProxyServer { SystemTime::now(), false, ); - let add_return_route_sub = self.out_subs("ProxyServer").add_return_route.clone(); let route_source = self.out_subs("Neighborhood").route_source.clone(); let proxy_server_sub = self.out_subs("ProxyServer").route_result_sub.clone(); let inbound_client_data_helper = self @@ -325,13 +301,7 @@ impl ProxyServer { .as_ref() .expect("IBCDHelper uninitialized"); - inbound_client_data_helper.request_route_and_transmit( - args, - add_return_route_sub, - route_source, - proxy_server_sub, - ); - retry + inbound_client_data_helper.request_route_and_transmit(args, route_source, proxy_server_sub); } fn retire_stream_key(&mut self, stream_key: &StreamKey) { @@ -342,7 +312,7 @@ impl ProxyServer { &self, client_addr: SocketAddr, proxy_protocol: ProxyProtocol, - hostname_opt: Option, + hostname: String, ) { self.subs .as_ref() @@ -351,119 +321,261 @@ impl ProxyServer { .try_send(TransmitDataMsg { endpoint: Endpoint::Socket(client_addr), last_data: true, - sequence_number: Some(0), // DNS resolution errors always happen on the first request + sequence_number_opt: Some(0), // DNS resolution errors always happen on the first request data: from_protocol(proxy_protocol) .server_impersonator() - .dns_resolution_failure_response(hostname_opt), + .dns_resolution_failure_response(hostname), }) .expect("Dispatcher is dead"); } - fn handle_dns_resolve_failure(&mut self, msg: &ExpiredCoresPackage) { - let return_route_info = - match self.get_return_route_info(&msg.remaining_route, "dns resolve failure") { - Some(rri) => rri, - None => return, // TODO: Eventually we'll have to do something better here, but we'll probably need some heuristics. - }; - let exit_public_key = { - // ugly, ugly - let self_public_key = self.cryptde_pair.main.as_ref().public_key(); - return_route_info - .find_exit_node_key() - .unwrap_or_else(|| { - if !self.is_decentralized { - self_public_key - } else { - panic!( - "Internal error: return_route_info for {} has no exit Node", - return_route_info.return_route_id + fn get_response_services( + route_query_response: &RouteQueryResponse, + ) -> Option<&[ExpectedService]> { + match &route_query_response.expected_services { + ExpectedServices::RoundTrip(_, back) => Some(back), + _ => None, + } + } + + fn find_exit_node_key(response_services: &[ExpectedService]) -> Option { + response_services + .iter() + .find_map(|service| service.exit_node_key_opt()) + } + + fn handle_add_route_result_message(&mut self, msg: AddRouteResultMessage) { + // We can't access self.logger for logging once we obtain mutable access to a stream_info + // element. So we create a delayed_log closure that we can call with self.logger after + // we've finished with the mutable borrow. We have to use #[allow(unused_assignments)] + // because Rust can't figure out that delayed_log will always be assigned before it's used. + type DelayedLogArgs = Box; + #[allow(unused_assignments)] + let mut delayed_log: DelayedLogArgs = Box::new(|_, _, _, _, _| {}); + let (target_hostname, stream_key, retries_left, message) = { + let mut stream_info = self.stream_info_mut(&msg.stream_key).unwrap_or_else(|| { + panic!( + "AddRouteResultMessage Handler: stream key: {} not found", + msg.stream_key + ) + }); + let dns_failure_retry = stream_info + .dns_failure_retry_opt + .as_ref() + .unwrap_or_else(|| + panic!("AddRouteResultMessage Handler: dns_failure_retry_opt is None for stream key {}", msg.stream_key) + ); + let mut message = String::new(); + match msg.result { + Ok(route_query_response) => { + delayed_log = Box::new( + move |logger: &Logger, + target_hostname: String, + stream_key: StreamKey, + retries_left: usize, + _: String| { + debug!( + logger, + "Found a new route for hostname: {:?} - stream key: {} retries left: {}", + target_hostname, + stream_key, + retries_left ); - } - }) - .clone() + }, + ); + stream_info.route_opt = Some(route_query_response); + } + Err(e) => { + message = e; + delayed_log = Box::new( + move |logger: &Logger, + target_hostname: String, + stream_key: StreamKey, + retries_left: usize, + message: String| { + warning!( + logger, + "No route found for hostname: {:?} - stream key {} - retries left: {} - AddRouteResultMessage Error: {}", + target_hostname, + stream_key, + retries_left, + message + ); + }, + ); + } + } + ( + dns_failure_retry + .unsuccessful_request + .target_hostname + .clone(), + msg.stream_key, + dns_failure_retry.retries_left, + message, + ) }; + delayed_log( + &self.logger, + target_hostname, + stream_key, + retries_left, + message, + ); + } - let hostname_opt = return_route_info.hostname_opt.clone(); + fn handle_dns_resolve_failure(&mut self, msg: &ExpiredCoresPackage) { let response = &msg.payload; - match self.keys_and_addrs.a_to_b(&response.stream_key) { - Some(client_addr) => { - if let Some(server_name) = hostname_opt.clone() { - self.subs - .as_ref() - .expect("Neighborhood unbound in ProxyServer") - .update_node_record_metadata - .try_send(UpdateNodeRecordMetadataMessage { - public_key: exit_public_key, - metadata_change: NRMetadataChange::AddUnreachableHost { - hostname: server_name, - }, - }) - .expect("Neighborhood is dead"); - } else { + // The idea here is that the Borrow Checker will not allow us to modify the StreamInfo in + // ProxyServer's map while we're looking at one of its values. So we're making a mutable + // copy of the StreamInfo, modifying that as necessary, and then overwriting the original + // map element with the modified copy at the end of the function. However, under certain + // circumstances we want to _retire_ the stream key; so we have a restore_stream_info + // flag that starts out true and is set to false if we retire the stream key. It's an + // ugly hack. Thanks, Borrow Checker! + let mut stream_info = match self.stream_info.remove(&response.stream_key) { + Some(info) => info, + None => { + error!( + self.logger, + "Discarding DnsResolveFailure message from an unrecognized stream key {:?}", + &response.stream_key + ); + return; + } + }; + let mut restore_stream_info = true; + + let route_query_response = match &stream_info.route_opt { + Some(route_query_response) => route_query_response, + None => { + error!( + self.logger, + "Stream info for stream key {} has no route info", &response.stream_key + ); + return; + } + }; + let response_services = match Self::get_response_services(route_query_response) { + Some(response_services) => response_services, + None => { + error!( + self.logger, + "Stream info for stream key {} has no response services in its route info", + &response.stream_key + ); + return; + } + }; + let exit_public_key = if !self.is_decentralized { + self.cryptde_pair.main.public_key().clone() + } else { + match Self::find_exit_node_key(response_services) { + Some(exit_public_key) => exit_public_key, + None => { error!( self.logger, - "Exit node {exit_public_key} complained of DNS failure, but was given no hostname to resolve." + "Stream info for stream key {} has no exit node in its response services", + &response.stream_key ); - // TODO: Malefactor ban the exit node because it lied about the DNS failure. + return; } - self.report_response_services_consumed(&return_route_info, 0, msg.payload_len); - let retry = match self.remove_dns_failure_retry(&response.stream_key) { - Ok(retry) => retry, - Err(error_msg) => { - error!( - self.logger, - "While handling ExpiredCoresPackage: {}", error_msg + } + }; + let response = &msg.payload; + + match self.keys_and_addrs.a_to_b(&response.stream_key) { + Some(client_addr) => { + self.subs + .as_ref() + .expect("Neighborhood unbound in ProxyServer") + .update_node_record_metadata + .try_send(UpdateNodeRecordMetadataMessage { + public_key: exit_public_key, + metadata_change: NRMetadataChange::AddUnreachableHost { + hostname: route_query_response.host.name.clone(), + }, + }) + .expect("Neighborhood is dead"); + self.report_response_services_consumed(response_services, 0, msg.payload_len); + if let Some(retry_ref) = &mut stream_info.dns_failure_retry_opt { + debug!( + self.logger, + "Handling DNS failure for hostname {:?} - stream key: {} retries left: {}", + retry_ref.unsuccessful_request.target_hostname, + &response.stream_key, + retry_ref.retries_left + ); + if retry_ref.retries_left > 0 { + self.retry_dns_resolution(retry_ref, client_addr); + retry_ref.retries_left -= 1; + } else { + restore_stream_info = false; + self.retire_stream_key(&response.stream_key); + let protocol = stream_info.protocol_opt.expect( + "StreamInfo should always have a protocol_opt set by the time we get a DNS failure" + ); + self.send_dns_failure_response_to_the_browser( + client_addr, + protocol, + route_query_response.host.name.clone(), ); - return; } - }; - if retry.retries_left > 0 { - let mut returned_retry = self.retry_dns_resolution(retry, client_addr); - returned_retry.retries_left -= 1; - self.dns_failure_retries - .insert(response.stream_key, returned_retry); } else { - self.retire_stream_key(&response.stream_key); - self.send_dns_failure_response_to_the_browser( - client_addr, - return_route_info.protocol, - hostname_opt, + error!( + self.logger, + "While handling ExpiredCoresPackage: No DNSFailureRetry entry found for the stream_key: {:?}", + &response.stream_key ); + return; } } None => { error!(self.logger, "Discarding DnsResolveFailure message for {} from an unrecognized stream key {:?}", - hostname_opt.unwrap_or_else(|| "".to_string()), + route_query_response.host.name, &response.stream_key ) } } + if restore_stream_info { + self.stream_info.insert(response.stream_key, stream_info); + } } fn schedule_stream_key_purge(&mut self, stream_key: StreamKey) { - let host_info = match self.tunneled_hosts.get(&stream_key) { - None => String::from(""), - Some(hostname) => format!(", which was tunneling to the host {:?}", hostname), - }; - debug!( - self.logger, - "Client closed stream referenced by stream key {:?}{}. It will be purged after {:?}.", - &stream_key, - host_info, - self.stream_key_purge_delay - ); - self.stream_key_ttl.insert(stream_key, SystemTime::now()); - self.subs - .as_ref() - .expect("ProxyServer Subs Unbound") - .schedule_stream_key_purge - .try_send(MessageScheduler { - scheduled_msg: StreamKeyPurge { stream_key }, - delay: self.stream_key_purge_delay, - }) - .expect("ProxyServer is dead"); + let stream_key_purge_delay = self.stream_key_purge_delay; + // We can't access self.logger for logging once we obtain mutable access to a stream_info + // element. So we create a delayed_log closure that we can call with self.logger after + // we've finished with the mutable borrow. + let mut delayed_log: Box = Box::new(|_: &Logger| {}); + if let Some(stream_info) = self.stream_info_mut(&stream_key) { + let host_info = match &stream_info.tunneled_host_opt { + None => String::from(""), + Some(hostname) => format!(", which was tunneling to the host {:?}", hostname), + }; + delayed_log = Box::new(move |logger: &Logger| { + debug!( + logger, + "Client closed stream referenced by stream key {:?}{}. It will be purged after {:?}.", + &stream_key, + host_info, + stream_key_purge_delay + ); + }); + stream_info.time_to_live_opt = Some(SystemTime::now()); + self.subs + .as_ref() + .expect("ProxyServer Subs Unbound") + .schedule_stream_key_purge + .try_send(MessageScheduler { + scheduled_msg: StreamKeyPurge { stream_key }, + delay: self.stream_key_purge_delay, + }) + .expect("ProxyServer is dead"); + } + delayed_log(&self.logger); } fn log_straggling_packet( @@ -488,14 +600,6 @@ impl ProxyServer { &mut self, msg: ExpiredCoresPackage, ) { - debug!( - self.logger, - "ExpiredCoresPackage remaining_route: {}", - msg.remaining_route.to_string(vec![ - self.cryptde_pair.main.as_ref(), - self.cryptde_pair.main.as_ref() - ]) - ); let payload_data_len = msg.payload_len; let response = msg.payload; debug!( @@ -503,67 +607,69 @@ impl ProxyServer { "Relaying ClientResponsePayload (stream key {}, sequence {}, length {}) from Hopper to Dispatcher for client", response.stream_key, response.sequenced_packet.sequence_number, response.sequenced_packet.data.len() ); - let return_route_info = - match self.get_return_route_info(&msg.remaining_route, "client response") { - Some(rri) => rri, - None => return, - }; + let expected_services = match self.get_expected_return_services(&response.stream_key) { + Some(expected_services) => expected_services, + None => return, + }; self.report_response_services_consumed( - &return_route_info, + &expected_services, response.sequenced_packet.data.len(), payload_data_len, ); let stream_key = response.stream_key; - match self.remove_dns_failure_retry(&stream_key) { - Ok(_) => { - debug!(self.logger, "Successful attempt of DNS resolution, removing DNS retry entry for stream key: {}", &response.stream_key) - } - Err(_) => { + if let Some(info) = self.stream_info_mut(&stream_key) { + if let Err(e) = ProxyServer::remove_dns_failure_retry(info, &stream_key) { trace!( self.logger, - "No DNS retry entry found for stream key: {} during a successful attempt", - &response.stream_key - ) + "No DNS retry entry found for stream key {} during a successful attempt: {}", + &stream_key, + e + ); } } - if let Some(old_timestamp) = self.stream_key_ttl.get(&stream_key) { - self.log_straggling_packet(&stream_key, payload_data_len, old_timestamp) - } else { - match self.keys_and_addrs.a_to_b(&stream_key) { - Some(socket_addr) => { - let last_data = response.sequenced_packet.last_data; - let sequence_number = Some( - response.sequenced_packet.sequence_number - + self.browser_proxy_sequence_offset as u64, - ); - self.subs - .as_ref() - .expect("Dispatcher unbound in ProxyServer") - .dispatcher - .try_send(TransmitDataMsg { - endpoint: Endpoint::Socket(socket_addr), - last_data, - sequence_number, - data: response.sequenced_packet.data, - }) - .expect("Dispatcher is dead"); - if last_data { - self.purge_stream_key(&stream_key, "last data received from the exit node"); + if let Some(info) = self.stream_info(&stream_key) { + if let Some(old_timestamp) = info.time_to_live_opt { + self.log_straggling_packet(&stream_key, payload_data_len, &old_timestamp) + } else { + match self.keys_and_addrs.a_to_b(&stream_key) { + Some(socket_addr) => { + let last_data = response.sequenced_packet.last_data; + let sequence_number_opt = Some( + response.sequenced_packet.sequence_number + + self.browser_proxy_sequence_offset as u64, + ); + self.subs + .as_ref() + .expect("Dispatcher unbound in ProxyServer") + .dispatcher + .try_send(TransmitDataMsg { + endpoint: Endpoint::Socket(socket_addr), + last_data, + sequence_number_opt, + data: response.sequenced_packet.data, + }) + .expect("Dispatcher is dead"); + if last_data { + self.purge_stream_key( + &stream_key, + "last data received from the exit node", + ); + } + } + None => { + // TODO GH-608: It would be really nice to be able to send an InboundClientData with last_data: true + // back to the ProxyClient (and the distant server) so that the server could shut down + // its stream, since the browser has shut down _its_ stream and no more data will + // ever be accepted from the server on that stream; but we don't have enough information + // to do so, since our stream key has been purged and all the information it keyed + // is gone. Sorry, server! + warning!(self.logger, + "Discarding {}-byte packet {} from an unrecognized stream key: {:?}; can't send response back to client", + response.sequenced_packet.data.len(), + response.sequenced_packet.sequence_number, + response.stream_key, + ) } - } - None => { - // TODO GH-608: It would be really nice to be able to send an InboundClientData with last_data: true - // back to the ProxyClient (and the distant server) so that the server could shut down - // its stream, since the browser has shut down _its_ stream and no more data will - // ever be accepted from the server on that stream; but we don't have enough information - // to do so, since our stream key has been purged and all the information it keyed - // is gone. Sorry, server! - warning!(self.logger, - "Discarding {}-byte packet {} from an unrecognized stream key: {:?}; can't send response back to client", - response.sequenced_packet.data.len(), - response.sequenced_packet.sequence_number, - response.stream_key, - ) } } } @@ -574,7 +680,10 @@ impl ProxyServer { match http_data { Some(ref host) if host.port == TLS_PORT => { let stream_key = self.find_or_generate_stream_key(msg); - self.tunneled_hosts.insert(stream_key, host.name.clone()); + match self.stream_info_mut(&stream_key) { + None => return, + Some(stream_info) => stream_info.tunneled_host_opt = Some(host.name.clone()), + } self.subs .as_ref() .expect("Dispatcher unbound in ProxyServer") @@ -582,7 +691,7 @@ impl ProxyServer { .try_send(TransmitDataMsg { endpoint: Endpoint::Socket(msg.client_addr), last_data: false, - sequence_number: msg.sequence_number, + sequence_number_opt: msg.sequence_number_opt, data: b"HTTP/1.1 200 OK\r\n\r\n".to_vec(), }) .expect("Dispatcher is dead"); @@ -595,7 +704,7 @@ impl ProxyServer { .try_send(TransmitDataMsg { endpoint: Endpoint::Socket(msg.client_addr), last_data: true, - sequence_number: msg.sequence_number, + sequence_number_opt: msg.sequence_number_opt, data: b"HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\n\r\n".to_vec(), }) .expect("Dispatcher is dead"); @@ -636,14 +745,13 @@ impl ProxyServer { let ibcd = InboundClientData { timestamp: SystemTime::now(), client_addr: msg.peer_addr, - reception_port: Some(nca.reception_port), + reception_port_opt: Some(nca.reception_port), last_data: true, is_clandestine: false, - sequence_number: Some(nca.sequence_number), + sequence_number_opt: Some(nca.sequence_number), data: vec![], }; - if let Err(e) = - self.help(|helper, proxy| helper.handle_normal_client_data(proxy, ibcd, true)) + if let Err(e) = self.help(|helper, proxy| helper.handle_normal_client_data(proxy, ibcd)) { error!(self.logger, "{}", e) }; @@ -667,6 +775,16 @@ impl ProxyServer { ibcd.client_addr, ); self.keys_and_addrs.insert(stream_key, ibcd.client_addr); + self.stream_info.insert( + stream_key, + StreamInfo { + tunneled_host_opt: None, + dns_failure_retry_opt: None, + route_opt: None, + protocol_opt: None, + time_to_live_opt: None, + }, + ); debug!( self.logger, "find_or_generate_stream_key() inserted new key {} for {}", @@ -684,9 +802,7 @@ impl ProxyServer { "Retiring stream key {} due to {}", &stream_key, reason ); let _ = self.keys_and_addrs.remove_a(stream_key); - let _ = self.stream_key_routes.remove(stream_key); - let _ = self.tunneled_hosts.remove(stream_key); - let _ = self.stream_key_ttl.remove(stream_key); + let _ = self.stream_info.remove(stream_key); } fn make_payload( @@ -694,10 +810,17 @@ impl ProxyServer { ibcd: InboundClientData, stream_key: &StreamKey, ) -> Result { - let tunnelled_host = self.tunneled_hosts.get(stream_key); - let new_ibcd = match tunnelled_host { + let stream_info_opt = self.stream_info.get(stream_key); + let (host_opt, tunnelled_host_opt) = match stream_info_opt { + None => (None, None), + Some(info) => match &info.route_opt { + Some(route) => (Some(route.host.clone()), info.tunneled_host_opt.clone()), + None => (None, info.tunneled_host_opt.clone()), + }, + }; + let new_ibcd = match tunnelled_host_opt { Some(_) => InboundClientData { - reception_port: Some(443), + reception_port_opt: Some(TLS_PORT), ..ibcd }, None => ibcd, @@ -705,13 +828,14 @@ impl ProxyServer { match self.client_request_payload_factory.make( &new_ibcd, *stream_key, + host_opt, self.cryptde_pair.alias.as_ref(), &self.logger, ) { None => Err("Couldn't create ClientRequestPayload".to_string()), - Some(payload) => match tunnelled_host { + Some(payload) => match tunnelled_host_opt { Some(hostname) => Ok(ClientRequestPayload_0v1 { - target_hostname: Some(hostname.clone()), + target_hostname: hostname, ..payload }), None => Ok(payload), @@ -719,26 +843,19 @@ impl ProxyServer { } } + fn get_next_return_route_id(&self) -> u32 { + let return_route_id = self.next_return_route_id.get(); + self.next_return_route_id + .set(return_route_id.wrapping_add(1)); + return_route_id + } + fn try_transmit_to_hopper( args: TransmitToHopperArgs, - add_return_route_sub: Recipient, route_query_response: RouteQueryResponse, ) -> Result<(), String> { match route_query_response.expected_services { - ExpectedServices::RoundTrip(over, back, return_route_id) => { - let return_route_info = AddReturnRouteMessage { - return_route_id, - expected_services: back, - protocol: args.payload.protocol, - hostname_opt: args.payload.target_hostname.clone(), - }; - debug!( - args.logger, - "Adding expectant return route info: {:?}", return_route_info - ); - add_return_route_sub - .try_send(return_route_info) - .expect("ProxyServer is dead"); + ExpectedServices::RoundTrip(over, _) => { ProxyServer::transmit_to_hopper(args, route_query_response.route, over) } _ => panic!("Expected RoundTrip ExpectedServices but got OneWay"), @@ -891,7 +1008,7 @@ impl ProxyServer { source_addr: SocketAddr, dispatcher: &Recipient, ) -> String { - let target_hostname = ProxyServer::hostname(&payload); + let target_hostname = payload.target_hostname.clone(); let stream_key = payload.stream_key; ProxyServer::send_route_failure(payload, source_addr, dispatcher); format!( @@ -907,56 +1024,45 @@ impl ProxyServer { ) { let data = from_protocol(payload.protocol) .server_impersonator() - .route_query_failure_response(&ProxyServer::hostname(&payload)); + .route_query_failure_response(&payload.target_hostname); let msg = TransmitDataMsg { endpoint: Endpoint::Socket(source_addr), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data, }; dispatcher.try_send(msg).expect("Dispatcher is dead"); } - fn hostname(payload: &ClientRequestPayload_0v1) -> String { - match payload.target_hostname { - Some(ref thn) => thn.clone(), - None => "".to_string(), - } - } - - fn get_return_route_info( - &self, - remaining_route: &Route, - source: &str, - ) -> Option> { - let mut mut_remaining_route = remaining_route.clone(); - mut_remaining_route - .shift(self.cryptde_pair.main.as_ref()) - .expect("Internal error: remaining route in ProxyServer with no hops"); - let return_route_id = match mut_remaining_route.id(self.cryptde_pair.main.as_ref()) { - Ok(rri) => rri, - Err(e) => { - error!(self.logger, "Can't report services consumed: {}", e); - return None; - } - }; - match self.route_ids_to_return_routes.get(&return_route_id) { - Some(rri) => Some(rri), + fn get_expected_return_services( + &mut self, + stream_key: &StreamKey, + ) -> Option> { + match self.stream_info(stream_key) { None => { - error!(self.logger, "Can't report services consumed: received response with bogus return-route ID {} for {}. Ignoring", return_route_id, source); + error!(self.logger, "Can't pay for return services consumed: received response with unrecognized stream key {:?}. Ignoring", stream_key); None } + Some(stream_info) => { + let route = stream_info + .route_opt + .as_ref() + .unwrap_or_else(|| panic!("Internal error: Request was sent over stream {:?} without an associated route being stored in stream_info: can't pay", stream_key)); + match route.expected_services { + ExpectedServices::RoundTrip(_, ref return_services) => Some(return_services.clone()), + _ => panic!("Internal error: ExpectedServices in ProxyServer for stream key {:?} is not RoundTrip", stream_key), + } + } } } fn report_response_services_consumed( &self, - return_route_info: &AddReturnRouteMessage, + expected_services: &[ExpectedService], exit_size: usize, routing_size: usize, ) { - let exit_service_report: ExitServiceSearch = return_route_info - .expected_services + let exit_service_report: ExitServiceSearch = expected_services .iter() .filter(|service| !matches!(service, ExpectedService::Nothing)) .fold(ZeroHop, |acc, service| { @@ -982,8 +1088,7 @@ impl ProxyServer { ZeroHop => return, Definite(report) => report, }; - let routing_service_reports = return_route_info - .expected_services + let routing_service_reports = expected_services .iter() .flat_map(|service| match service { ExpectedService::Routing(_, wallet, rate_pack) => Some(RoutingServiceConsumed { @@ -1022,13 +1127,11 @@ pub trait IBCDHelper { &self, proxy_s: &mut ProxyServer, msg: InboundClientData, - retire_stream_key: bool, ) -> Result<(), String>; fn request_route_and_transmit( &self, args: TransmitToHopperArgs, - add_return_route_sub: Recipient, route_source: Recipient, proxy_server_sub: Recipient, ); @@ -1038,7 +1141,7 @@ trait RouteQueryResponseResolver: Send { fn resolve_message( &self, args: TransmitToHopperArgs, - add_return_route_sub: Recipient, + // add_return_route_sub: Recipient, proxy_server_sub: Recipient, route_result_opt: Result, MailboxError>, ); @@ -1049,18 +1152,13 @@ impl RouteQueryResponseResolver for RouteQueryResponseResolverReal { fn resolve_message( &self, args: TransmitToHopperArgs, - add_return_route_sub: Recipient, proxy_server_sub: Recipient, route_result_opt: Result, MailboxError>, ) { let stream_key = args.payload.stream_key; let result = match route_result_opt { Ok(Some(route_query_response)) => { - match ProxyServer::try_transmit_to_hopper( - args, - add_return_route_sub, - route_query_response.clone(), - ) { + match ProxyServer::try_transmit_to_hopper(args, route_query_response.clone()) { Ok(()) => Ok(route_query_response), Err(e) => Err(e), } @@ -1102,15 +1200,16 @@ impl IBCDHelperReal { } } } + impl IBCDHelper for IBCDHelperReal { fn handle_normal_client_data( &self, - proxy: &mut ProxyServer, + proxy_server: &mut ProxyServer, msg: InboundClientData, - retire_stream_key: bool, ) -> Result<(), String> { let client_addr = msg.client_addr; - if proxy.consuming_wallet_balance.is_none() && proxy.is_decentralized { + let last_data = msg.last_data; + if proxy_server.consuming_wallet_balance.is_none() && proxy_server.is_decentralized { let protocol_pack = match from_ibcd(&msg) { Err(e) => return Err(e), Ok(pp) => pp, @@ -1121,25 +1220,25 @@ impl IBCDHelper for IBCDHelperReal { let msg = TransmitDataMsg { endpoint: Endpoint::Socket(client_addr), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data, }; - proxy + proxy_server .out_subs("Dispatcher") .dispatcher .try_send(msg) .expect("Dispatcher is dead"); return Err("Browser request rejected due to missing consuming wallet".to_string()); } - let stream_key = proxy.find_or_generate_stream_key(&msg); + let stream_key = proxy_server.find_or_generate_stream_key(&msg); let timestamp = msg.timestamp; - let payload = match proxy.make_payload(msg, &stream_key) { + let payload = match proxy_server.make_payload(msg, &stream_key) { Ok(payload) => { - if !proxy.is_running_in_integration_test { - if let Some(hostname) = &payload.target_hostname { - if let Err(e) = Hostname::new(hostname).validate_non_loopback_host() { - return Err(format!("Request to wildcard IP detected - {} (Most likely because Blockchain Service URL is not set)", e)); - } + if !proxy_server.is_running_in_integration_test { + if let Err(e) = + Hostname::new(&payload.target_hostname).validate_non_loopback_host() + { + return Err(format!("Request to wildcard IP detected - {} (Most likely because Blockchain Service URL is not set)", e)); } } payload @@ -1147,38 +1246,47 @@ impl IBCDHelper for IBCDHelperReal { Err(e) => return Err(e), }; - if proxy.dns_failure_retries.get(&stream_key).is_none() { - let dns_failure_retry = DNSFailureRetry { - unsuccessful_request: payload.clone(), - retries_left: if proxy.is_decentralized { 3 } else { 0 }, - }; - proxy - .dns_failure_retries - .insert(stream_key, dns_failure_retry); + { + let is_decentralized = proxy_server.is_decentralized; + let mut stream_info = proxy_server + .stream_info_mut(&stream_key) + .unwrap_or_else(|| panic!("Stream key {} disappeared!", &stream_key)); + if stream_info.dns_failure_retry_opt.is_none() { + let dns_failure_retry = DNSFailureRetry { + unsuccessful_request: payload.clone(), + retries_left: if is_decentralized { + DNS_FAILURE_RETRIES + } else { + 0 + }, + }; + stream_info.dns_failure_retry_opt = Some(dns_failure_retry); + stream_info.protocol_opt = Some(payload.protocol); + } } let args = - TransmitToHopperArgs::new(proxy, payload, client_addr, timestamp, retire_stream_key); - let add_return_route_sub = proxy.out_subs("ProxysServer").add_return_route.clone(); + TransmitToHopperArgs::new(proxy_server, payload, client_addr, timestamp, last_data); let pld = &args.payload; - if let Some(route_query_response) = proxy.stream_key_routes.get(&pld.stream_key) { + let stream_info = proxy_server + .stream_info(&pld.stream_key) + .unwrap_or_else(|| panic!("Stream key {} disappeared!", &pld.stream_key)); + if let Some(route_query_response) = &stream_info.route_opt { debug!( - proxy.logger, + proxy_server.logger, "Transmitting down existing stream {}: sequence {}, length {}", pld.stream_key, pld.sequenced_packet.sequence_number, pld.sequenced_packet.data.len() ); let route_query_response = route_query_response.clone(); - ProxyServer::try_transmit_to_hopper(args, add_return_route_sub, route_query_response) + ProxyServer::try_transmit_to_hopper(args, route_query_response) } else { - let route_source = proxy.out_subs("Neighborhood").route_source.clone(); - let proxy_server_sub = proxy.out_subs("ProxyServer").route_result_sub.clone(); - self.request_route_and_transmit( - args, - add_return_route_sub, - route_source, - proxy_server_sub, - ); + let route_source = proxy_server.out_subs("Neighborhood").route_source.clone(); + let proxy_server_sub = proxy_server + .out_subs("ProxyServer") + .route_result_sub + .clone(); + self.request_route_and_transmit(args, route_source, proxy_server_sub); Ok(()) } } @@ -1186,12 +1294,11 @@ impl IBCDHelper for IBCDHelperReal { fn request_route_and_transmit( &self, args: TransmitToHopperArgs, - add_return_route_sub: Recipient, neighborhood_sub: Recipient, proxy_server_sub: Recipient, ) { let pld = &args.payload; - let hostname_opt = pld.target_hostname.clone(); + let host = Host::new(&pld.target_hostname, pld.target_port); let logger = args.logger.clone(); debug!( logger, @@ -1206,16 +1313,11 @@ impl IBCDHelper for IBCDHelperReal { tokio::spawn( neighborhood_sub .send(RouteQueryMessage::data_indefinite_route_request( - hostname_opt, + host, payload_size, )) .then(move |route_result| { - message_resolver.resolve_message( - args, - add_return_route_sub, - proxy_server_sub, - route_result, - ); + message_resolver.resolve_message(args, proxy_server_sub, route_result); Ok(()) }), ); @@ -1225,6 +1327,7 @@ impl IBCDHelper for IBCDHelperReal { pub struct TransmitToHopperArgs { pub main_cryptde: Box, pub payload: ClientRequestPayload_0v1, + pub return_route_id: u32, pub client_addr: SocketAddr, pub timestamp: SystemTime, pub is_decentralized: bool, @@ -1253,9 +1356,11 @@ impl TransmitToHopperArgs { } else { None }; + let return_route_id = proxy_server.get_next_return_route_id(); Self { main_cryptde: proxy_server.cryptde_pair.main.dup(), payload, + return_route_id, client_addr, timestamp, logger: proxy_server.logger.clone(), @@ -1354,6 +1459,7 @@ impl Hostname { #[cfg(test)] mod tests { use super::*; + use crate::blockchain::bip32::Bip32EncryptionKeyProvider; use crate::bootstrapper::CryptDEPair; use crate::match_lazily_every_type_id; use crate::proxy_server::protocol_pack::ServerImpersonator; @@ -1367,6 +1473,7 @@ mod tests { use crate::sub_lib::dispatcher::Component; use crate::sub_lib::hop::LiveHop; use crate::sub_lib::hopper::MessageType; + use crate::sub_lib::host::Host; use crate::sub_lib::neighborhood::ExpectedServices; use crate::sub_lib::neighborhood::{ExpectedService, DEFAULT_RATE_PACK}; use crate::sub_lib::proxy_client::{ClientResponsePayload_0v1, DnsResolveFailure_0v1}; @@ -1375,7 +1482,6 @@ mod tests { use crate::sub_lib::route::Route; use crate::sub_lib::route::RouteSegment; use crate::sub_lib::sequence_buffer::SequencedPacket; - use crate::sub_lib::ttl_hashmap::TtlHashMap; use crate::sub_lib::versioned_data::VersionedData; use crate::test_utils::make_meaningless_route; use crate::test_utils::make_paying_wallet; @@ -1390,7 +1496,6 @@ mod tests { }; use crate::test_utils::zero_hop_route_response; use actix::System; - use crossbeam_channel::unbounded; use lazy_static::lazy_static; use masq_lib::constants::{HTTP_PORT, TLS_PORT}; use masq_lib::test_utils::logging::init_test_logging; @@ -1459,7 +1564,6 @@ mod tests { fn resolve_message( &self, args: TransmitToHopperArgs, - _add_return_route_sub: Recipient, _proxy_server_sub: Recipient, route_result: Result, MailboxError>, ) { @@ -1486,11 +1590,59 @@ mod tests { self } } + + struct StreamInfoBuilder { + product: StreamInfo, + } + + impl StreamInfoBuilder { + pub fn new() -> Self { + Self { + product: StreamInfo { + tunneled_host_opt: None, + dns_failure_retry_opt: None, + route_opt: None, + protocol_opt: None, + time_to_live_opt: None, + }, + } + } + + pub fn tunneled_host(mut self, host: &str) -> Self { + self.product.tunneled_host_opt = Some(host.to_string()); + self + } + + pub fn dns_failure_retry(mut self, retry: DNSFailureRetry) -> Self { + self.product.dns_failure_retry_opt = Some(retry); + self + } + + pub fn route(mut self, route: RouteQueryResponse) -> Self { + self.product.route_opt = Some(route); + self + } + + pub fn protocol(mut self, protocol: ProxyProtocol) -> Self { + self.product.protocol_opt = Some(protocol); + self + } + + pub fn time_to_live(mut self, ttl: SystemTime) -> Self { + self.product.time_to_live_opt = Some(ttl); + self + } + + pub fn build(self) -> StreamInfo { + self.product + } + } + #[test] fn constants_have_correct_values() { assert_eq!(CRASH_KEY, "PROXYSERVER"); - assert_eq!(RETURN_ROUTE_TTL, Duration::from_secs(120)); assert_eq!(STREAM_KEY_PURGE_DELAY, Duration::from_secs(30)); + assert_eq!(DNS_FAILURE_RETRIES, 3); } const STANDARD_CONSUMING_WALLET_BALANCE: i64 = 0; @@ -1504,7 +1656,6 @@ mod tests { accountant: recipient!(addr, ReportServicesConsumedMessage), route_source: recipient!(addr, RouteQueryMessage), update_node_record_metadata: recipient!(addr, UpdateNodeRecordMetadataMessage), - add_return_route: recipient!(addr, AddReturnRouteMessage), stream_shutdown_sub: recipient!(addr, StreamShutdownMsg), route_result_sub: recipient!(addr, AddRouteResultMessage), schedule_stream_key_purge: recipient!(addr, MessageScheduler), @@ -1548,16 +1699,9 @@ mod tests { } } - fn return_route_with_id(cryptde: &dyn CryptDE, return_route_id: u32) -> Route { - let cover_hop = make_cover_hop(cryptde); - let id_hop = cryptde - .encode( - &cryptde.public_key(), - &PlainData::from(serde_cbor::ser::to_vec(&return_route_id).unwrap()), - ) - .unwrap(); + fn return_route(cryptde: &dyn CryptDE) -> Route { Route { - hops: vec![cover_hop, id_hop], + hops: vec![make_cover_hop(cryptde)], } } @@ -1576,7 +1720,7 @@ mod tests { #[derive(Default)] struct IBCDHelperMock { - handle_normal_client_data_params: Arc>>, + handle_normal_client_data_params: Arc>>, handle_normal_client_data_results: RefCell>>, } @@ -1585,12 +1729,11 @@ mod tests { &self, _proxy_s: &mut ProxyServer, msg: InboundClientData, - retire_stream_key: bool, ) -> Result<(), String> { self.handle_normal_client_data_params .lock() .unwrap() - .push((msg, retire_stream_key)); + .push(msg); self.handle_normal_client_data_results .borrow_mut() .remove(0) @@ -1599,7 +1742,6 @@ mod tests { fn request_route_and_transmit( &self, _args: TransmitToHopperArgs, - _add_return_route_sub: Recipient, _route_source: Recipient, _proxy_server_sub: Recipient, ) { @@ -1610,7 +1752,7 @@ mod tests { impl IBCDHelperMock { fn handle_normal_client_data_params( mut self, - params: &Arc>>, + params: &Arc>>, ) -> Self { self.handle_normal_client_data_params = params.clone(); self @@ -1625,47 +1767,125 @@ mod tests { } #[test] - fn proxy_server_receives_http_request_with_new_stream_key_from_dispatcher_then_sends_cores_package_to_hopper( - ) { - init_test_logging(); - let test_name = "proxy_server_receives_http_request_with_new_stream_key_from_dispatcher_then_sends_cores_package_to_hopper"; - let main_cryptde = CRYPTDE_PAIR.main.as_ref(); - let alias_cryptde = CRYPTDE_PAIR.alias.as_ref(); - let http_request = b"GET /index.html HTTP/1.1\r\nHost: nowhere.com\r\n\r\n"; - let (hopper_mock, hopper_awaiter, hopper_log_arc) = make_recorder(); - let (neighborhood_mock, _, neighborhood_recording_arc) = make_recorder(); - let destination_key = PublicKey::from(&b"our destination"[..]); - let neighborhood_mock = neighborhood_mock.route_query_response(Some(RouteQueryResponse { - route: Route { hops: vec![] }, - expected_services: ExpectedServices::RoundTrip( - vec![make_exit_service_from_key(destination_key.clone())], - vec![], - 1234, - ), - })); - let (proxy_server_mock, _, proxy_server_recording_arc) = make_recorder(); - let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); + fn get_expected_services_produces_nothing_if_nothing_exists() { + let mut subject = ProxyServer::new( + CRYPTDE_PAIR.clone(), + true, + Some(STANDARD_CONSUMING_WALLET_BALANCE), + false, + false, + ); let stream_key = StreamKey::make_meaningless_stream_key(); - let expected_data = http_request.to_vec(); - let msg_from_dispatcher = InboundClientData { - timestamp: SystemTime::now(), - client_addr: socket_addr, - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), - last_data: true, - is_clandestine: false, - data: expected_data.clone(), - }; - let expected_http_request = PlainData::new(http_request); - let route = Route { hops: vec![] }; - let expected_payload = ClientRequestPayload_0v1 { + + let result = subject.get_expected_return_services(&stream_key); + + assert!( + result.is_none(), + "Expected no expected services, but got: {:?}", + result + ); + } + + #[test] + fn get_expected_services_produces_rri_when_it_exists() { + let mut subject = ProxyServer::new( + CRYPTDE_PAIR.clone(), + true, + Some(STANDARD_CONSUMING_WALLET_BALANCE), + false, + false, + ); + let exit_public_key = PublicKey::new(&b"exit key"[..]); + let stream_key = StreamKey::make_meaningless_stream_key(); + let back_services = vec![ExpectedService::Exit( + exit_public_key, + make_wallet("booga"), + rate_pack(1000), + )]; + let expected_services = ExpectedServices::RoundTrip(vec![], back_services.clone()); + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: make_meaningless_route(&CRYPTDE_PAIR), + expected_services: expected_services.clone(), + host: Host::new("booga.com", TLS_PORT), + }) + .protocol(ProxyProtocol::TLS) + .build(), + ); + + let result = subject.get_expected_return_services(&stream_key).unwrap(); + + assert_eq!(result, back_services); + } + + #[test] + #[should_panic( + expected = "Internal error: Request was sent over stream Y29uc3RhbnQgZm9yIHBhbmljIG0 without an associated route being stored in stream_info: can't pay" + )] + fn get_expected_services_panics_if_stream_info_exists_but_has_no_route() { + let mut subject = ProxyServer::new( + CRYPTDE_PAIR.clone(), + true, + Some(STANDARD_CONSUMING_WALLET_BALANCE), + false, + false, + ); + let stream_key = StreamKey::from_bytes(b"constant for panic message"); + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + // no route_opt: problem + .protocol(ProxyProtocol::TLS) + .build(), + ); + + let _ = subject.get_expected_return_services(&stream_key).unwrap(); + } + + #[test] + fn proxy_server_receives_http_request_with_new_stream_key_from_dispatcher_then_sends_cores_package_to_hopper( + ) { + init_test_logging(); + let test_name = "proxy_server_receives_http_request_with_new_stream_key_from_dispatcher_then_sends_cores_package_to_hopper"; + let main_cryptde = CRYPTDE_PAIR.main.as_ref(); + let alias_cryptde = CRYPTDE_PAIR.alias.as_ref(); + let http_request = b"GET /index.html HTTP/1.1\r\nHost: nowhere.com\r\n\r\n"; + let (hopper_mock, hopper_awaiter, hopper_log_arc) = make_recorder(); + let (neighborhood_mock, _, neighborhood_recording_arc) = make_recorder(); + let destination_key = PublicKey::from(&b"our destination"[..]); + let neighborhood_mock = neighborhood_mock.route_query_response(Some(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![make_exit_service_from_key(destination_key.clone())], + vec![], + ), + host: Host::new("booga.com", HTTP_PORT), + })); + let (proxy_server_mock, _, proxy_server_recording_arc) = make_recorder(); + let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); + let stream_key = StreamKey::make_meaningless_stream_key(); + let expected_data = http_request.to_vec(); + let msg_from_dispatcher = InboundClientData { + timestamp: SystemTime::now(), + client_addr: socket_addr, + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), + last_data: true, + is_clandestine: false, + data: expected_data.clone(), + }; + let expected_http_request = PlainData::new(http_request); + let route = Route { hops: vec![] }; + let expected_payload = ClientRequestPayload_0v1 { stream_key: stream_key.clone(), sequenced_packet: SequencedPacket { data: expected_http_request.into(), sequence_number: 0, last_data: true, }, - target_hostname: Some(String::from("nowhere.com")), + target_hostname: String::from("nowhere.com"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: alias_cryptde.public_key().clone(), @@ -1682,7 +1902,7 @@ mod tests { thread::spawn(move || { let stream_key_factory = StreamKeyFactoryMock::new() .make_parameters(&make_parameters_arc) - .make_result(stream_key); + .make_result(stream_key.clone()); let system = System::new(test_name); let mut subject = ProxyServer::new( CRYPTDE_PAIR.clone(), @@ -1704,6 +1924,13 @@ mod tests { subject_addr.try_send(msg_from_dispatcher).unwrap(); + subject_addr + .try_send(AssertionsMessage { + assertions: Box::new(move |proxy_server: &mut ProxyServer| { + assert!(proxy_server.stream_info.contains_key(&stream_key)); + }), + }) + .unwrap(); system.run(); }); @@ -1720,13 +1947,16 @@ mod tests { let record = recording.get_record::(0); assert_eq!( record, - &RouteQueryMessage::data_indefinite_route_request(Some("nowhere.com".to_string()), 47) + &RouteQueryMessage::data_indefinite_route_request( + Host::new("nowhere.com", HTTP_PORT), + 47 + ) ); let recording = proxy_server_recording_arc.lock().unwrap(); assert_eq!(recording.len(), 0); TestLogHandler::new().exists_log_containing( - &format!("DEBUG: {test_name}: Found a new route for hostname: Some(\"nowhere.com\") - stream key: {stream_key} retries left: 3") + &format!("DEBUG: {test_name}: Found a new route for hostname: \"nowhere.com\" - stream key: {stream_key} retries left: 3") ); } @@ -1743,19 +1973,20 @@ mod tests { expected_services: ExpectedServices::RoundTrip( vec![make_exit_service_from_key(destination_key.clone())], vec![], - 1234, ), + host: Host::new("booga.com", HTTP_PORT), })); let route = Route { hops: vec![] }; let (dispatcher_mock, _, dispatcher_recording_arc) = make_recorder(); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); let stream_key = StreamKey::make_meaningless_stream_key(); let request_data = http_request.to_vec(); + let tunneled_data = make_server_com_client_hello(); let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(8443), - sequence_number: Some(0), + reception_port_opt: Some(8443), + sequence_number_opt: Some(0), last_data: false, is_clandestine: false, data: request_data.clone(), @@ -1763,27 +1994,27 @@ mod tests { let tunnelled_msg = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr, - reception_port: Some(8443), - sequence_number: Some(0), + reception_port_opt: Some(8443), + sequence_number_opt: Some(0), last_data: false, is_clandestine: false, - data: b"client hello".to_vec(), + data: tunneled_data.clone(), }; let expected_tdm = TransmitDataMsg { endpoint: Endpoint::Socket(socket_addr), last_data: false, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: b"HTTP/1.1 200 OK\r\n\r\n".to_vec(), }; let expected_payload = ClientRequestPayload_0v1 { stream_key: stream_key.clone(), sequenced_packet: SequencedPacket { - data: b"client hello".to_vec(), + data: tunneled_data, sequence_number: 0, last_data: false, }, - target_hostname: Some(String::from("realdomain.nu")), - target_port: 443, + target_hostname: String::from("realdomain.nu"), + target_port: TLS_PORT, protocol: ProxyProtocol::TLS, originator_public_key: alias_cryptde.public_key().clone(), }; @@ -1800,7 +2031,7 @@ mod tests { thread::spawn(move || { let stream_key_factory = StreamKeyFactoryMock::new() .make_parameters(&make_parameters_arc_thread) - .make_result(stream_key); + .make_result(stream_key.clone()); let system = System::new( "proxy_server_receives_connect_responds_with_ok_and_stores_stream_key_and_hostname", ); @@ -1822,6 +2053,13 @@ mod tests { subject_addr.try_send(msg_from_dispatcher).unwrap(); subject_addr.try_send(tunnelled_msg).unwrap(); + subject_addr + .try_send(AssertionsMessage { + assertions: Box::new(move |proxy_server: &mut ProxyServer| { + assert!(proxy_server.stream_info.contains_key(&stream_key)); + }), + }) + .unwrap(); system.run(); }); @@ -1844,8 +2082,8 @@ mod tests { assert_eq!( neighborhood_record, &RouteQueryMessage::data_indefinite_route_request( - Some("realdomain.nu".to_string()), - 12 + Host::new("realdomain.nu", TLS_PORT), + 68 ) ); } @@ -1868,15 +2106,18 @@ mod tests { subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); - - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![ExpectedService::Nothing], - protocol: ProxyProtocol::TLS, - hostname_opt: None, - }, + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: make_meaningless_route(&CRYPTDE_PAIR), + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ExpectedService::Nothing], + ), + host: Host::new("booga.com", TLS_PORT), + }) + .build(), ); let subject_addr: Addr = subject.start(); let http_request = b"CONNECT https://realdomain.nu:443 HTTP/1.1\r\nHost: https://bunkjunk.wrong:443\r\n\r\n"; @@ -1884,10 +2125,10 @@ mod tests { let inbound_client_data = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr, - reception_port: Some(443), + reception_port_opt: Some(TLS_PORT), last_data: false, is_clandestine: false, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: request_data, }; @@ -1904,7 +2145,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), client_response_payload.into(), 0, ); @@ -1921,9 +2162,10 @@ mod tests { system.run(); let dispatcher_recording = dispatcher_log_arc.lock().unwrap(); + let record = dispatcher_recording.get_record::(0); + assert_eq!(record.sequence_number_opt.unwrap(), 0); let record = dispatcher_recording.get_record::(1); - - assert_eq!(record.sequence_number.unwrap(), 1); + assert_eq!(record.sequence_number_opt.unwrap(), 1); } #[test] @@ -1936,7 +2178,7 @@ mod tests { let (dispatcher_mock, _dispatcher_awaiter, dispatcher_recording_arc) = make_recorder(); let neighborhood_mock = neighborhood_mock.route_query_response(Some( - zero_hop_route_response(&cryptde.public_key(), cryptde), + zero_hop_route_response(&cryptde.public_key(), cryptde, false), )); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); @@ -1945,8 +2187,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(8443), - sequence_number: Some(0), + reception_port_opt: Some(8443), + sequence_number_opt: Some(0), last_data: false, is_clandestine: false, data: request_data.clone(), @@ -1987,7 +2229,7 @@ mod tests { let expected_transmit_data_msg = TransmitDataMsg { endpoint: Endpoint::Socket(socket_addr), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: b"HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\n\r\n".to_vec(), }; @@ -2007,7 +2249,7 @@ mod tests { let (dispatcher_mock, _dispatcher_awaiter, dispatcher_recording_arc) = make_recorder(); let neighborhood_mock = neighborhood_mock.route_query_response(Some( - zero_hop_route_response(&cryptde.public_key(), cryptde), + zero_hop_route_response(&cryptde.public_key(), cryptde, false), )); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); @@ -2016,8 +2258,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(8443), - sequence_number: Some(0), + reception_port_opt: Some(8443), + sequence_number_opt: Some(0), last_data: false, is_clandestine: false, data: request_data.clone(), @@ -2058,7 +2300,7 @@ mod tests { let expected_transmit_data_msg = TransmitDataMsg { endpoint: Endpoint::Socket(socket_addr), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: b"HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\n\r\n".to_vec(), }; @@ -2082,8 +2324,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data.clone(), @@ -2117,7 +2359,7 @@ mod tests { &TransmitDataMsg { endpoint: Endpoint::Socket(socket_addr), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: server_impersonator.consuming_wallet_absent(), } ); @@ -2140,8 +2382,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(TLS_PORT), - sequence_number: Some(0), + reception_port_opt: Some(TLS_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data.clone(), @@ -2175,7 +2417,7 @@ mod tests { &TransmitDataMsg { endpoint: Endpoint::Socket(socket_addr), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: server_impersonator.consuming_wallet_absent(), } ); @@ -2192,7 +2434,8 @@ mod tests { let alias_cryptde = CRYPTDE_PAIR.alias.as_ref(); let expected_data = b"GET /index.html HTTP/1.1\r\nHost: nowhere.com\r\n\r\n".to_vec(); let expected_data_inner = expected_data.clone(); - let expected_route = zero_hop_route_response(main_cryptde.public_key(), main_cryptde); + let expected_route = + zero_hop_route_response(main_cryptde.public_key(), main_cryptde, false); let stream_key = StreamKey::make_meaningless_stream_key(); let (hopper, hopper_awaiter, hopper_log_arc) = make_recorder(); let neighborhood = Recorder::new().route_query_response(Some(expected_route.clone())); @@ -2203,8 +2446,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data_inner, @@ -2213,7 +2456,12 @@ mod tests { let system = System::new("proxy_server_receives_http_request_with_no_consuming_wallet_in_zero_hop_mode_and_handles_normally"); let mut subject = ProxyServer::new(CRYPTDE_PAIR.clone(), false, None, false, false); subject.stream_key_factory = Box::new(stream_key_factory); - subject.keys_and_addrs.insert(stream_key, socket_addr); + subject + .keys_and_addrs + .insert(stream_key.clone(), socket_addr); + subject + .stream_info + .insert(stream_key, StreamInfoBuilder::new().build()); let subject_addr: Addr = subject.start(); let peer_actors = peer_actors_builder() .dispatcher(dispatcher) @@ -2235,7 +2483,7 @@ mod tests { target_component: Component::ProxyClient, return_component_opt: Some(Component::ProxyServer), payload_size: 47, - hostname_opt: Some("nowhere.com".to_string()), + host: Host::new("nowhere.com", HTTP_PORT), } ); let dispatcher_recording = dispatcher_log_arc.lock().unwrap(); @@ -2251,8 +2499,8 @@ mod tests { &ClientRequestPayload_0v1 { stream_key, sequenced_packet: SequencedPacket::new(expected_data, 0, true), - target_hostname: Some("nowhere.com".to_string()), - target_port: 80, + target_hostname: "nowhere.com".to_string(), + target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: alias_cryptde.public_key().clone(), } @@ -2271,28 +2519,35 @@ mod tests { let alias_cryptde = CRYPTDE_PAIR.alias.as_ref(); let expected_data = b"Fake TLS request".to_vec(); let expected_data_inner = expected_data.clone(); - let expected_route = zero_hop_route_response(main_cryptde.public_key(), main_cryptde); + let expected_route = zero_hop_route_response(main_cryptde.public_key(), main_cryptde, true); + let expected_route_inner = expected_route.clone(); let stream_key = StreamKey::make_meaningless_stream_key(); let (hopper, hopper_awaiter, hopper_log_arc) = make_recorder(); let neighborhood = Recorder::new().route_query_response(Some(expected_route.clone())); - let neighborhood_log_arc = neighborhood.get_recording(); let (dispatcher, _, dispatcher_log_arc) = make_recorder(); thread::spawn(move || { let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(TLS_PORT), - sequence_number: Some(0), + reception_port_opt: Some(TLS_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data_inner, }; - let stream_key_factory = StreamKeyFactoryMock::new(); // can't make any stream keys; shouldn't have to + let stream_key_factory = StreamKeyFactoryMock::new().make_result(stream_key.clone()); let system = System::new("proxy_server_receives_tls_request_with_no_consuming_wallet_in_zero_hop_mode_and_handles_normally"); let mut subject = ProxyServer::new(CRYPTDE_PAIR.clone(), false, None, false, false); subject.stream_key_factory = Box::new(stream_key_factory); subject.keys_and_addrs.insert(stream_key, socket_addr); + subject.stream_info.insert( + stream_key, + StreamInfoBuilder::new() + .route(expected_route_inner) + .protocol(ProxyProtocol::TLS) + .build(), + ); let subject_addr: Addr = subject.start(); let peer_actors = peer_actors_builder() .dispatcher(dispatcher) @@ -2306,17 +2561,6 @@ mod tests { system.run(); }); hopper_awaiter.await_message_count(1); - let neighborhood_recording = neighborhood_log_arc.lock().unwrap(); - assert_eq!( - neighborhood_recording.get_record::(0), - &RouteQueryMessage { - target_key_opt: None, - target_component: Component::ProxyClient, - return_component_opt: Some(Component::ProxyServer), - payload_size: 16, - hostname_opt: None, - } - ); let dispatcher_recording = dispatcher_log_arc.lock().unwrap(); assert!(dispatcher_recording.is_empty()); let hopper_recording = hopper_log_arc.lock().unwrap(); @@ -2330,8 +2574,8 @@ mod tests { &ClientRequestPayload_0v1 { stream_key, sequenced_packet: SequencedPacket::new(expected_data, 0, true), - target_hostname: None, - target_port: 443, + target_hostname: "booga.com".to_string(), + target_port: TLS_PORT, protocol: ProxyProtocol::TLS, originator_public_key: alias_cryptde.public_key().clone(), } @@ -2357,8 +2601,8 @@ mod tests { expected_services: ExpectedServices::RoundTrip( vec![make_exit_service_from_key(destination_key.clone())], vec![], - 1234, ), + host: Host::new("booga.com", HTTP_PORT), })); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); let stream_key = StreamKey::make_meaningless_stream_key(); @@ -2366,8 +2610,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data.clone(), @@ -2381,7 +2625,7 @@ mod tests { sequence_number: 0, last_data: true, }, - target_hostname: Some(String::from("nowhere.com")), + target_hostname: String::from("nowhere.com"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: alias_cryptde.public_key().clone(), @@ -2405,6 +2649,9 @@ mod tests { ); subject.stream_key_factory = Box::new(stream_key_factory); subject.keys_and_addrs.insert(stream_key, socket_addr); + subject + .stream_info + .insert(stream_key.clone(), StreamInfoBuilder::new().build()); let subject_addr: Addr = subject.start(); let peer_actors = peer_actors_builder() .hopper(hopper_mock) @@ -2456,7 +2703,6 @@ mod tests { ), main_cryptde, Some(consuming_wallet), - 1234, Some(TEST_DEFAULT_CHAIN.rec().contract), ) .unwrap(); @@ -2476,8 +2722,8 @@ mod tests { ExpectedService::Nothing, ExpectedService::Exit(PublicKey::new(&[3]), earning_wallet, rate_pack(102)), ], - 1234, ), + host: Host::new("booga.com", HTTP_PORT), })); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); let stream_key = StreamKey::make_meaningless_stream_key(); @@ -2485,8 +2731,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data.clone(), @@ -2499,7 +2745,7 @@ mod tests { sequence_number: 0, last_data: true, }, - target_hostname: Some(String::from("nowhere.com")), + target_hostname: String::from("nowhere.com"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: alias_cryptde.public_key().clone(), @@ -2542,7 +2788,10 @@ mod tests { let record = recording.get_record::(0); assert_eq!( record, - &RouteQueryMessage::data_indefinite_route_request(Some("nowhere.com".to_string()), 47) + &RouteQueryMessage::data_indefinite_route_request( + Host::new("nowhere.com", HTTP_PORT), + 47 + ) ); } @@ -2560,8 +2809,8 @@ mod tests { expected_services: ExpectedServices::RoundTrip( vec![expected_service.clone()], vec![expected_service], - 123, ), + host: Host::new("booga.com", HTTP_PORT), }); let (neighborhood_mock, _, _) = make_recorder(); let neighborhood_mock = @@ -2572,8 +2821,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data.clone(), @@ -2632,8 +2881,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data.clone(), @@ -2684,8 +2933,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data.clone(), @@ -2719,13 +2968,14 @@ mod tests { let alias_cryptde = CRYPTDE_PAIR.alias.as_ref(); let http_request = b"GET /index.html HTTP/1.1\r\nHost: nowhere.com\r\n\r\n"; let destination_key = PublicKey::from(&b"our destination"[..]); + let route = Route { hops: vec![] }; let route_query_response = RouteQueryResponse { - route: Route { hops: vec![] }, + route: route.clone(), expected_services: ExpectedServices::RoundTrip( vec![make_exit_service_from_key(destination_key.clone())], vec![], - 1234, ), + host: Host::new("booga.com", HTTP_PORT), }; let (hopper_mock, hopper_awaiter, hopper_recording_arc) = make_recorder(); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); @@ -2733,9 +2983,9 @@ mod tests { let expected_data = http_request.to_vec(); let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), - client_addr: socket_addr.clone(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), + client_addr: socket_addr, + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data.clone(), @@ -2747,14 +2997,14 @@ mod tests { sequence_number: 0, last_data: true, }, - target_hostname: Some(String::from("nowhere.com")), + target_hostname: String::from("nowhere.com"), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: alias_cryptde.public_key().clone(), }; let expected_pkg = IncipientCoresPackage::new( main_cryptde, - Route { hops: vec![] }, + route, expected_payload.into(), &destination_key, ) @@ -2772,8 +3022,13 @@ mod tests { ); subject.stream_key_factory = Box::new(stream_key_factory); subject - .stream_key_routes - .insert(stream_key, route_query_response); + .keys_and_addrs + .insert(stream_key.clone(), socket_addr); + subject.stream_info.insert( + stream_key, + StreamInfoBuilder::new().route(route_query_response).build(), + ); + subject.next_return_route_id = Cell::new(4444); let subject_addr: Addr = subject.start(); let peer_actors = peer_actors_builder().hopper(hopper_mock).build(); subject_addr.try_send(BindMessage { peer_actors }).unwrap(); @@ -2793,57 +3048,89 @@ mod tests { fn proxy_server_sends_message_to_accountant_about_all_services_consumed_on_the_route_over() { let cryptde = CRYPTDE_PAIR.main.as_ref(); let now = SystemTime::now(); - let exit_earning_wallet = make_wallet("exit earning wallet"); - let route_1_earning_wallet = make_wallet("route 1 earning wallet"); - let route_2_earning_wallet = make_wallet("route 2 earning wallet"); + let routing_node_1_public_key = PublicKey::new(&[1]); + let routing_node_2_public_key = PublicKey::new(&[2]); + let exit_node_public_key = PublicKey::new(&[3]); + let key_bytes = b"__originating consuming wallet__"; + let keypair = Bip32EncryptionKeyProvider::from_raw_secret(key_bytes).unwrap(); + let originating_consuming_wallet = Wallet::from(keypair); + let routing_node_1_earning_wallet = make_wallet("route 1 earning wallet"); + let routing_node_2_earning_wallet = make_wallet("route 2 earning wallet"); + let exit_node_earning_wallet = make_wallet("exit earning wallet"); + let routing_node_1_rate_pack = rate_pack(101); + let routing_node_2_rate_pack = rate_pack(102); + let exit_node_rate_pack = rate_pack(103); let http_request = b"GET /index.html HTTP/1.1\r\nHost: nowhere.com\r\n\r\n"; let (accountant_mock, _, accountant_recording_arc) = make_recorder(); let (hopper_mock, _, hopper_recording_arc) = make_recorder(); let (proxy_server_mock, _, proxy_server_recording_arc) = make_recorder(); - let routing_node_1_rate_pack = rate_pack(101); - let routing_node_2_rate_pack = rate_pack(102); - let exit_node_rate_pack = rate_pack(103); + let over_route_segment = RouteSegment::new( + vec![ + &cryptde.public_key(), + &routing_node_1_public_key, + &routing_node_2_public_key, + &exit_node_public_key, + ], + Component::ProxyClient, + ); + let back_route_segment = RouteSegment::new( + vec![ + &exit_node_public_key, + &routing_node_2_public_key, + &routing_node_1_public_key, + &cryptde.public_key(), + ], + Component::ProxyServer, + ); + let route = Route::round_trip( + over_route_segment, + back_route_segment, + cryptde, + Some(originating_consuming_wallet), + Some(TEST_DEFAULT_CHAIN.rec().contract), + ) + .unwrap(); let route_query_response = RouteQueryResponse { - route: make_meaningless_route(&CRYPTDE_PAIR), + route, expected_services: ExpectedServices::RoundTrip( vec![ ExpectedService::Nothing, ExpectedService::Routing( - PublicKey::new(&[1]), - route_1_earning_wallet.clone(), - routing_node_1_rate_pack, + routing_node_1_public_key.clone(), + routing_node_1_earning_wallet.clone(), + routing_node_1_rate_pack.clone(), ), ExpectedService::Routing( - PublicKey::new(&[2]), - route_2_earning_wallet.clone(), - routing_node_2_rate_pack, + routing_node_2_public_key.clone(), + routing_node_2_earning_wallet.clone(), + routing_node_2_rate_pack.clone(), ), ExpectedService::Exit( - PublicKey::new(&[3]), - exit_earning_wallet.clone(), - exit_node_rate_pack, + exit_node_public_key.clone(), + exit_node_earning_wallet.clone(), + exit_node_rate_pack.clone(), ), ], vec![ ExpectedService::Exit( - PublicKey::new(&[3]), - make_wallet("some wallet 1"), - rate_pack(104), + exit_node_public_key.clone(), + exit_node_earning_wallet.clone(), + exit_node_rate_pack, ), ExpectedService::Routing( - PublicKey::new(&[2]), - make_wallet("some wallet 2"), - rate_pack(105), + routing_node_2_public_key.clone(), + routing_node_2_earning_wallet.clone(), + routing_node_2_rate_pack, ), ExpectedService::Routing( - PublicKey::new(&[1]), - make_wallet("some wallet 3"), - rate_pack(106), + routing_node_1_public_key.clone(), + routing_node_1_earning_wallet.clone(), + routing_node_1_rate_pack, ), ExpectedService::Nothing, ], - 0, ), + host: Host::new("booga.com", HTTP_PORT), }; let source_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); let stream_key = StreamKey::make_meaningless_stream_key(); @@ -2859,7 +3146,7 @@ mod tests { let payload = ClientRequestPayload_0v1 { stream_key, sequenced_packet: SequencedPacket::new(expected_data, 0, false), - target_hostname: Some("nowhere.com".to_string()), + target_hostname: "nowhere.com".to_string(), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(b"originator_public_key"), @@ -2868,6 +3155,7 @@ mod tests { let args = TransmitToHopperArgs { main_cryptde: cryptde.dup(), payload, + return_route_id: 4444, client_addr: source_addr, timestamp: now, is_decentralized: true, @@ -2878,17 +3166,35 @@ mod tests { retire_stream_key_sub_opt: None, }; - let result = ProxyServer::try_transmit_to_hopper( - args, - peer_actors.proxy_server.add_return_route, - route_query_response, - ); + let result = ProxyServer::try_transmit_to_hopper(args, route_query_response); System::current().stop(); system.run(); let recording = hopper_recording_arc.lock().unwrap(); - let record = recording.get_record::(0); + let mut record = recording.get_record::(0).clone(); let payload_enc_length = record.payload.len(); + let _ = record.route.shift(cryptde); + let _ = record.route.shift(&CryptDENull::from( + &routing_node_1_public_key, + TEST_DEFAULT_CHAIN, + )); + let _ = record.route.shift(&CryptDENull::from( + &routing_node_2_public_key, + TEST_DEFAULT_CHAIN, + )); + let _ = record.route.shift(&CryptDENull::from( + &exit_node_public_key, + TEST_DEFAULT_CHAIN, + )); + let _ = record.route.shift(&CryptDENull::from( + &routing_node_2_public_key, + TEST_DEFAULT_CHAIN, + )); + let _ = record.route.shift(&CryptDENull::from( + &routing_node_1_public_key, + TEST_DEFAULT_CHAIN, + )); + let _ = record.route.shift(cryptde); let recording = accountant_recording_arc.lock().unwrap(); let record = recording.get_record::(0); assert_eq!(recording.len(), 1); @@ -2897,7 +3203,7 @@ mod tests { &ReportServicesConsumedMessage { timestamp: now, exit: ExitServiceConsumed { - earning_wallet: exit_earning_wallet, + earning_wallet: exit_node_earning_wallet, payload_size: exit_payload_size, service_rate: exit_node_rate_pack.exit_service_rate, byte_rate: exit_node_rate_pack.exit_byte_rate @@ -2905,12 +3211,12 @@ mod tests { routing_payload_size: payload_enc_length, routing: vec![ RoutingServiceConsumed { - earning_wallet: route_1_earning_wallet, + earning_wallet: routing_node_1_earning_wallet, service_rate: routing_node_1_rate_pack.routing_service_rate, byte_rate: routing_node_1_rate_pack.routing_byte_rate, }, RoutingServiceConsumed { - earning_wallet: route_2_earning_wallet, + earning_wallet: routing_node_2_earning_wallet, service_rate: routing_node_2_rate_pack.routing_service_rate, byte_rate: routing_node_2_rate_pack.routing_byte_rate, } @@ -2918,8 +3224,7 @@ mod tests { } ); let recording = proxy_server_recording_arc.lock().unwrap(); - let _ = recording.get_record::(0); // don't care about this, other than type - assert_eq!(recording.len(), 1); // No StreamShutdownMsg: that's the important thing + assert_eq!(recording.len(), 0); // No StreamShutdownMsg: that's the important thing assert_eq!(result, Ok(())); } @@ -2933,8 +3238,8 @@ mod tests { expected_services: ExpectedServices::RoundTrip( vec![ExpectedService::Nothing], vec![ExpectedService::Nothing], - 0, ), + host: Host::new("booga.com", HTTP_PORT), }; let source_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); let stream_key = StreamKey::make_meaningless_stream_key(); @@ -2947,7 +3252,7 @@ mod tests { let payload = ClientRequestPayload_0v1 { stream_key, sequenced_packet: SequencedPacket::new(expected_data, 0, false), - target_hostname: Some("nowhere.com".to_string()), + target_hostname: "nowhere.com".to_string(), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(b"originator_public_key"), @@ -2956,6 +3261,7 @@ mod tests { let args = TransmitToHopperArgs { main_cryptde: cryptde.dup(), payload, + return_route_id: 3333, client_addr: source_addr, timestamp: SystemTime::now(), is_decentralized: false, @@ -2966,26 +3272,12 @@ mod tests { retire_stream_key_sub_opt: Some(peer_actors.proxy_server.stream_shutdown_sub), }; - let result = ProxyServer::try_transmit_to_hopper( - args, - peer_actors.proxy_server.add_return_route, - route_query_response, - ); + let result = ProxyServer::try_transmit_to_hopper(args, route_query_response); System::current().stop(); system.run(); let recording = proxy_server_recording_arc.lock().unwrap(); - let record = recording.get_record::(0); - assert_eq!( - record, - &AddReturnRouteMessage { - return_route_id: 0, - expected_services: vec![ExpectedService::Nothing], - protocol: ProxyProtocol::HTTP, - hostname_opt: Some("nowhere.com".to_string()) - } - ); - let record = recording.get_record::(1); + let record = recording.get_record::(0); assert_eq!( record, &StreamShutdownMsg { @@ -3007,7 +3299,8 @@ mod tests { let http_request = b"GET /index.html HTTP/1.1\r\nHost: nowhere.com\r\n\r\n"; let (accountant_mock, accountant_awaiter, _) = make_recorder(); let (neighborhood_mock, _, _) = make_recorder(); - let mut route_query_response = zero_hop_route_response(&cryptde.public_key(), cryptde); + let mut route_query_response = + zero_hop_route_response(&cryptde.public_key(), cryptde, false); route_query_response.expected_services = ExpectedServices::RoundTrip( vec![ExpectedService::Exit( cryptde.public_key().clone(), @@ -3015,7 +3308,6 @@ mod tests { rate_pack(3), )], vec![], - 0, ); let neighborhood_mock = neighborhood_mock.route_query_response(Some(route_query_response.clone())); @@ -3025,8 +3317,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data.clone(), @@ -3061,28 +3353,35 @@ mod tests { #[test] #[should_panic( - expected = "AddRouteResultMessage Handler: stream key: AAAAAAAAAAAAAAAAAAAAAAAAAAA not found within dns_failure_retries" + expected = "AddRouteResultMessage Handler: dns_failure_retry_opt is None for stream key" )] - fn route_result_message_handler_panics_when_dns_retries_hashmap_doesnt_contain_a_stream_key() { - let system = System::new("route_result_message_handler_panics_when_dns_retries_hashmap_doesnt_contain_a_stream_key"); - let subject = ProxyServer::new( + fn route_result_message_handler_panics_when_no_dns_retries_exist() { + init_test_logging(); + let system = System::new("route_result_message_handler_panics_when_no_dns_retries_exist"); + let mut subject = ProxyServer::new( CRYPTDE_PAIR.clone(), true, Some(STANDARD_CONSUMING_WALLET_BALANCE), false, false, ); + let stream_key = StreamKey::make_meaningless_stream_key(); + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new().build(), // no DNS retries + ); let subject_addr: Addr = subject.start(); let peer_actors = peer_actors_builder().build(); subject_addr.try_send(BindMessage { peer_actors }).unwrap(); subject_addr .try_send(AddRouteResultMessage { - stream_key: StreamKey::make_meaningless_stream_key(), + stream_key, result: Err("Some Error".to_string()), }) .unwrap(); + System::current().stop(); system.run(); } @@ -3144,8 +3443,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), last_data: true, data: expected_data.clone(), is_clandestine: false, @@ -3181,7 +3480,7 @@ mod tests { let expected_msg = TransmitDataMsg { endpoint: Endpoint::Socket(SocketAddr::from_str("1.2.3.4:5678").unwrap()), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: ServerImpersonatorHttp {}.route_query_failure_response("nowhere.com"), }; assert_eq!(record, &expected_msg); @@ -3189,10 +3488,13 @@ mod tests { let record = recording.get_record::(0); assert_eq!( record, - &RouteQueryMessage::data_indefinite_route_request(Some("nowhere.com".to_string()), 47) + &RouteQueryMessage::data_indefinite_route_request( + Host::new("nowhere.com", HTTP_PORT), + 47 + ) ); TestLogHandler::new().exists_log_containing(&format!( - "WARN: {test_name}: No route found for hostname: Some(\"nowhere.com\") - stream key {stream_key} - retries left: 3 - AddRouteResultMessage Error: Failed to find route to nowhere.com" + "WARN: {test_name}: No route found for hostname: \"nowhere.com\" - stream key {stream_key} - retries left: 3 - AddRouteResultMessage Error: Failed to find route to nowhere.com" )); } @@ -3223,6 +3525,7 @@ mod tests { rate_pack(103), ), ]), + host: Host::new("booga.com", HTTP_PORT), }; let payload = ClientRequestPayload_0v1 { stream_key: StreamKey::make_meaningless_stream_key(), @@ -3231,7 +3534,7 @@ mod tests { sequence_number: 0, last_data: false, }, - target_hostname: None, + target_hostname: "booga.com".to_string(), target_port: 0, protocol: ProxyProtocol::TLS, originator_public_key: cryptde.public_key().clone(), @@ -3241,6 +3544,7 @@ mod tests { let args = TransmitToHopperArgs { main_cryptde: cryptde.dup(), payload, + return_route_id: 2222, client_addr: source_addr, timestamp: SystemTime::now(), is_decentralized: true, @@ -3251,11 +3555,7 @@ mod tests { retire_stream_key_sub_opt: None, }; - let _result = ProxyServer::try_transmit_to_hopper( - args, - peer_actors.proxy_server.add_return_route, - route_result, - ); + let _result = ProxyServer::try_transmit_to_hopper(args, route_result); } #[test] @@ -3269,25 +3569,20 @@ mod tests { false, false, ); - let add_return_route_message = AddReturnRouteMessage { - return_route_id: 0, - expected_services: vec![ - ExpectedService::Routing( - PublicKey::from(&b"key"[..]), - make_wallet("some wallet"), - rate_pack(10), - ), - ExpectedService::Exit( - PublicKey::from(&b"exit_key"[..]), - make_wallet("exit"), - rate_pack(11), - ), - ], - protocol: ProxyProtocol::HTTP, - hostname_opt: None, - }; + let expected_services = vec![ + ExpectedService::Routing( + PublicKey::from(&b"key"[..]), + make_wallet("some wallet"), + rate_pack(10), + ), + ExpectedService::Exit( + PublicKey::from(&b"exit_key"[..]), + make_wallet("exit"), + rate_pack(11), + ), + ]; - subject.report_response_services_consumed(&add_return_route_message, 1234, 3456); + subject.report_response_services_consumed(&expected_services, 1234, 3456); } #[test] @@ -3305,11 +3600,11 @@ mod tests { RouteSegment::new(vec![public_key, public_key], Component::ProxyServer), cryptde, None, - 1234, None, ) .unwrap(), - expected_services: ExpectedServices::RoundTrip(vec![], vec![], 1234), + expected_services: ExpectedServices::RoundTrip(vec![], vec![]), + host: Host::new("booga.com", HTTP_PORT), }; let neighborhood_mock = neighborhood_mock.route_query_response(Some(route_query_response)); let dispatcher = Recorder::new(); @@ -3320,8 +3615,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), last_data: true, data: expected_data.clone(), is_clandestine: false, @@ -3358,7 +3653,7 @@ mod tests { let expected_msg = TransmitDataMsg { endpoint: Endpoint::Socket(SocketAddr::from_str("1.2.3.4:5678").unwrap()), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: ServerImpersonatorHttp {}.route_query_failure_response("nowhere.com"), }; assert_eq!(record, &expected_msg); @@ -3366,35 +3661,19 @@ mod tests { let record = recording.get_record::(0); assert_eq!( record, - &RouteQueryMessage::data_indefinite_route_request(Some("nowhere.com".to_string()), 47) + &RouteQueryMessage::data_indefinite_route_request( + Host::new("nowhere.com", HTTP_PORT), + 47 + ) ); TestLogHandler::new().exists_log_containing(&format!( - "WARN: {test_name}: No route found for hostname: Some(\"nowhere.com\") - stream key {stream_key} - retries left: 3 - AddRouteResultMessage Error: Failed to find route to nowhere.com" + "WARN: {test_name}: No route found for hostname: \"nowhere.com\" - stream key {stream_key} - retries left: 3 - AddRouteResultMessage Error: Failed to find route to nowhere.com" )); } #[test] fn proxy_server_receives_tls_client_hello_from_dispatcher_then_sends_cores_package_to_hopper() { - let tls_request = &[ - 0x16, // content_type: Handshake - 0x00, 0x00, 0x00, 0x00, // version, length: don't care - 0x01, // handshake_type: ClientHello - 0x00, 0x00, 0x00, 0x00, 0x00, // length, version: don't care - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, // random: don't care - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, // random: don't care - 0x00, // session_id_length - 0x00, 0x00, // cipher_suites_length - 0x00, // compression_methods_length - 0x00, 0x13, // extensions_length - 0x00, 0x00, // extension_type: server_name - 0x00, 0x0F, // extension_length - 0x00, 0x0D, // server_name_list_length - 0x00, // server_name_type - 0x00, 0x0A, // server_name_length - b's', b'e', b'r', b'v', b'e', b'r', b'.', b'c', b'o', b'm', // server_name - ]; + let tls_request = make_server_com_client_hello(); let main_cryptde = CRYPTDE_PAIR.main.as_ref(); let alias_cryptde = CRYPTDE_PAIR.alias.as_ref(); let hopper_mock = Recorder::new(); @@ -3406,22 +3685,22 @@ mod tests { expected_services: ExpectedServices::RoundTrip( vec![make_exit_service_from_key(destination_key.clone())], vec![], - 1234, ), + host: Host::new("booga.com", TLS_PORT), })); let stream_key = StreamKey::make_meaningless_stream_key(); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); - let expected_data = tls_request.to_vec(); + let expected_data = tls_request.clone(); let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(TLS_PORT), - sequence_number: Some(0), + reception_port_opt: Some(TLS_PORT), + sequence_number_opt: Some(0), last_data: false, is_clandestine: false, data: expected_data.clone(), }; - let expected_tls_request = PlainData::new(tls_request); + let expected_tls_request = PlainData::new(tls_request.as_slice()); let route = Route { hops: vec![] }; let expected_payload = ClientRequestPayload_0v1 { stream_key: stream_key.clone(), @@ -3430,7 +3709,7 @@ mod tests { sequence_number: 0, last_data: false, }, - target_hostname: Some(String::from("server.com")), + target_hostname: String::from("server.com"), target_port: TLS_PORT, protocol: ProxyProtocol::TLS, originator_public_key: alias_cryptde.public_key().clone(), @@ -3491,8 +3770,8 @@ mod tests { expected_services: ExpectedServices::RoundTrip( vec![make_exit_service_from_key(destination_key.clone())], vec![], - 1234, ), + host: Host::new("booga.com", TLS_PORT), })); let stream_key = StreamKey::make_meaningless_stream_key(); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); @@ -3500,8 +3779,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(TLS_PORT), - sequence_number: Some(0), + reception_port_opt: Some(TLS_PORT), + sequence_number_opt: Some(0), last_data: false, is_clandestine: false, data: expected_data.clone(), @@ -3515,7 +3794,7 @@ mod tests { sequence_number: 0, last_data: false, }, - target_hostname: None, + target_hostname: "booga.com".to_string(), target_port: TLS_PORT, protocol: ProxyProtocol::TLS, originator_public_key: alias_cryptde.public_key().clone(), @@ -3537,6 +3816,22 @@ mod tests { ); subject.stream_key_factory = Box::new(StreamKeyFactoryMock::new().make_result(stream_key.clone())); + subject + .keys_and_addrs + .insert(stream_key.clone(), socket_addr); + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![make_exit_service_from_key(destination_key.clone())], + vec![], + ), + host: Host::new("booga.com", TLS_PORT), + }) + .build(), + ); let system = System::new("proxy_server_receives_tls_client_hello_from_dispatcher_then_sends_cores_package_to_hopper"); let subject_addr: Addr = subject.start(); let peer_actors = peer_actors_builder() @@ -3549,7 +3844,6 @@ mod tests { system.run(); }); - hopper_awaiter.await_message_count(1); let recording = hopper_log_arc.lock().unwrap(); let record = recording.get_record::(0); @@ -3575,8 +3869,8 @@ mod tests { expected_services: ExpectedServices::RoundTrip( vec![make_exit_service_from_key(destination_key.clone())], vec![], - 1234, ), + host: Host::new("booga.com", TLS_PORT), })); let client_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); let stream_key = StreamKey::make_meaningful_stream_key(test_name); @@ -3584,8 +3878,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr, - reception_port: Some(TLS_PORT), - sequence_number: Some(0), + reception_port_opt: Some(TLS_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data.clone(), @@ -3599,7 +3893,7 @@ mod tests { sequence_number: 0, last_data: true, }, - target_hostname: None, + target_hostname: "booga.com".to_string(), target_port: TLS_PORT, protocol: ProxyProtocol::TLS, originator_public_key: alias_cryptde.public_key().clone(), @@ -3620,6 +3914,20 @@ mod tests { false, ); subject.keys_and_addrs.insert(stream_key, client_addr); + subject.stream_info.insert( + stream_key, + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![make_exit_service_from_key(destination_key.clone())], + vec![], + ), + host: Host::new("booga.com", TLS_PORT), + }) + .protocol(ProxyProtocol::TLS) + .build(), + ); let system = System::new(test_name); let subject_addr: Addr = subject.start(); let peer_actors = peer_actors_builder() @@ -3673,8 +3981,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(TLS_PORT), - sequence_number: Some(0), + reception_port_opt: Some(TLS_PORT), + sequence_number_opt: Some(0), last_data: true, data: tls_request, is_clandestine: false, @@ -3708,7 +4016,7 @@ mod tests { let expected_msg = TransmitDataMsg { endpoint: Endpoint::Socket(SocketAddr::from_str("1.2.3.4:5678").unwrap()), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: ServerImpersonatorTls {}.route_query_failure_response("ignored"), }; assert_eq!(record, &expected_msg); @@ -3734,17 +4042,22 @@ mod tests { subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![ExpectedService::Nothing], - protocol: ProxyProtocol::TLS, - hostname_opt: None, - }, + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ExpectedService::Nothing], + ), + host: Host::new("booga.com", TLS_PORT), + }) + .protocol(ProxyProtocol::TLS) + .build(), ); let subject_addr: Addr = subject.start(); - let remaining_route = return_route_with_id(cryptde, 1234); + let remaining_route = return_route(cryptde); let client_response_payload = ClientResponsePayload_0v1 { stream_key: stream_key.clone(), sequenced_packet: SequencedPacket { @@ -3780,7 +4093,7 @@ mod tests { stream_key )); tlh.exists_log_containing(&format!( - "WARN: {test_name}: Discarding 16-byte packet 12345678 from an unrecognized stream key: {:?}", + "ERROR: {test_name}: Can't pay for return services consumed: received response with unrecognized stream key {:?}. Ignoring", stream_key )); } @@ -3819,24 +4132,17 @@ mod tests { subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); - subject.stream_key_routes.insert( + subject.stream_info.insert( stream_key.clone(), - RouteQueryResponse { - route: Route { hops: vec![] }, - expected_services: ExpectedServices::RoundTrip(vec![], vec![], 1234), - }, - ); - subject - .tunneled_hosts - .insert(stream_key.clone(), "hostname".to_string()); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![], - protocol: ProxyProtocol::HTTP, - hostname_opt: None, - }, + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip(vec![], vec![]), + host: Host::new("booga.com", HTTP_PORT), + }) + .protocol(ProxyProtocol::HTTP) + .tunneled_host("hostname") + .build(), ); let client_response_payload = ClientResponsePayload_0v1 { stream_key: stream_key.clone(), @@ -3849,7 +4155,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), client_response_payload.into(), 0, ); @@ -3857,8 +4163,7 @@ mod tests { subject.handle_client_response_payload(expired_cores_package); assert!(subject.keys_and_addrs.is_empty()); - assert!(subject.stream_key_routes.is_empty()); - assert!(subject.tunneled_hosts.is_empty()); + assert!(subject.stream_info.get(&stream_key).is_none()); } #[test] @@ -3926,24 +4231,17 @@ mod tests { subject .keys_and_addrs .insert(stream_key.clone(), msg.peer_addr.clone()); - subject.stream_key_routes.insert( + subject.stream_info.insert( stream_key.clone(), - RouteQueryResponse { - route: Route { hops: vec![] }, - expected_services: ExpectedServices::RoundTrip(vec![], vec![], 1234), - }, - ); - subject - .tunneled_hosts - .insert(stream_key.clone(), "hostname".to_string()); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![], - protocol: ProxyProtocol::HTTP, - hostname_opt: None, - }, + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip(vec![], vec![]), + host: Host::new("booga.com", HTTP_PORT), + }) + .protocol(ProxyProtocol::HTTP) + .tunneled_host("hostname") + .build(), ); let proxy_server_addr = subject.start(); let schedule_stream_key_purge_sub = proxy_server_addr.clone().recipient(); @@ -3961,18 +4259,14 @@ mod tests { .unwrap(); let pre_purge_assertions = AssertionsMessage { assertions: Box::new(move |proxy_server: &mut ProxyServer| { - let purge_timestamp = proxy_server - .stream_key_ttl - .get(&stream_key) - .unwrap() - .clone(); + let stream_info = proxy_server.stream_info.get(&stream_key).unwrap(); + let purge_timestamp = stream_info.time_to_live_opt.unwrap(); assert!( time_before_sending_package <= purge_timestamp && purge_timestamp <= time_after_sending_package ); + assert!(!proxy_server.stream_info.get(&stream_key).is_none()); assert!(!proxy_server.keys_and_addrs.is_empty()); - assert!(!proxy_server.stream_key_routes.is_empty()); - assert!(!proxy_server.tunneled_hosts.is_empty()); TestLogHandler::new().exists_log_containing(&format!( "DEBUG: {test_name}: Client closed stream referenced by stream key {:?}, \ which was tunneling to the host \"hostname\". \ @@ -3990,9 +4284,7 @@ mod tests { let post_purge_assertions = AssertionsMessage { assertions: Box::new(move |proxy_server: &mut ProxyServer| { assert!(proxy_server.keys_and_addrs.is_empty()); - assert!(proxy_server.stream_key_routes.is_empty()); - assert!(proxy_server.tunneled_hosts.is_empty()); - assert!(proxy_server.stream_key_ttl.is_empty()); + assert!(proxy_server.stream_info.get(&stream_key).is_none()); TestLogHandler::new().exists_log_containing(&format!( "DEBUG: {test_name}: Retiring stream key {:?}", stream_key @@ -4028,16 +4320,6 @@ mod tests { subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); - subject.stream_key_routes.insert( - stream_key.clone(), - RouteQueryResponse { - route: Route { hops: vec![] }, - expected_services: ExpectedServices::RoundTrip(vec![], vec![], 1234), - }, - ); - subject - .tunneled_hosts - .insert(stream_key.clone(), "hostname".to_string()); let exit_key = PublicKey::new(&b"blah"[..]); let exit_wallet = make_wallet("abc"); let exit_rates = RatePack { @@ -4046,22 +4328,26 @@ mod tests { exit_byte_rate: 100, exit_service_rate: 60000, }; - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![ExpectedService::Exit( - exit_key, - exit_wallet.clone(), - exit_rates.clone(), - )], - protocol: ProxyProtocol::HTTP, - hostname_opt: None, - }, + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ExpectedService::Exit( + exit_key, + exit_wallet.clone(), + exit_rates.clone(), + )], + ), + host: Host::new("booga.com", HTTP_PORT), + }) + .tunneled_host("hostname") + .protocol(ProxyProtocol::HTTP) + .time_to_live(SystemTime::now()) + .build(), ); - subject - .stream_key_ttl - .insert(stream_key.clone(), SystemTime::now()); let (accountant, _, accountant_recording_arc) = make_recorder(); let (dispatcher, _, dispatcher_recording_arc) = make_recorder(); let proxy_server_addr = subject.start(); @@ -4079,7 +4365,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), client_response_payload.into(), 5432, ); @@ -4137,63 +4423,35 @@ mod tests { let rate_pack_d = rate_pack(101); let rate_pack_e = rate_pack(102); let rate_pack_f = rate_pack(103); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![ - ExpectedService::Exit( - irrelevant_public_key.clone(), - incoming_route_d_wallet.clone(), - rate_pack_d, - ), - ExpectedService::Routing( - irrelevant_public_key.clone(), - incoming_route_e_wallet.clone(), - rate_pack_e, - ), - ExpectedService::Routing( - irrelevant_public_key.clone(), - incoming_route_f_wallet.clone(), - rate_pack_f, - ), - ExpectedService::Nothing, - ], - protocol: ProxyProtocol::TLS, - hostname_opt: None, - }, - ); - let incoming_route_g_wallet = make_wallet("G Earning"); - let incoming_route_h_wallet = make_wallet("H Earning"); - let incoming_route_i_wallet = make_wallet("I Earning"); - let rate_pack_g = rate_pack(104); - let rate_pack_h = rate_pack(105); - let rate_pack_i = rate_pack(106); - subject.route_ids_to_return_routes.insert( - 1235, - AddReturnRouteMessage { - return_route_id: 1235, - expected_services: vec![ - ExpectedService::Exit( - irrelevant_public_key.clone(), - incoming_route_g_wallet.clone(), - rate_pack_g, - ), - ExpectedService::Routing( - irrelevant_public_key.clone(), - incoming_route_h_wallet.clone(), - rate_pack_h, - ), - ExpectedService::Routing( - irrelevant_public_key.clone(), - incoming_route_i_wallet.clone(), - rate_pack_i, + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ + ExpectedService::Exit( + irrelevant_public_key.clone(), + incoming_route_d_wallet.clone(), + rate_pack_d, + ), + ExpectedService::Routing( + irrelevant_public_key.clone(), + incoming_route_e_wallet.clone(), + rate_pack_e, + ), + ExpectedService::Routing( + irrelevant_public_key.clone(), + incoming_route_f_wallet.clone(), + rate_pack_f, + ), + ExpectedService::Nothing, + ], ), - ExpectedService::Nothing, - ], - protocol: ProxyProtocol::TLS, - hostname_opt: None, - }, + host: Host::new("booga.com", HTTP_PORT), + }) + .build(), ); let subject_addr: Addr = subject.start(); let first_client_response_payload = ClientResponsePayload_0v1 { @@ -4209,7 +4467,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), first_client_response_payload.into(), 0, ); @@ -4227,7 +4485,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.5:1235").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1235), + return_route(cryptde), second_client_response_payload.into(), 0, ); @@ -4265,7 +4523,7 @@ mod tests { &ReportServicesConsumedMessage { timestamp: first_report_timestamp, exit: ExitServiceConsumed { - earning_wallet: incoming_route_d_wallet, + earning_wallet: incoming_route_d_wallet.clone(), payload_size: first_exit_size, service_rate: rate_pack_d.exit_service_rate, byte_rate: rate_pack_d.exit_byte_rate @@ -4273,12 +4531,12 @@ mod tests { routing_payload_size: routing_size, routing: vec![ RoutingServiceConsumed { - earning_wallet: incoming_route_e_wallet, + earning_wallet: incoming_route_e_wallet.clone(), service_rate: rate_pack_e.routing_service_rate, byte_rate: rate_pack_e.routing_byte_rate }, RoutingServiceConsumed { - earning_wallet: incoming_route_f_wallet, + earning_wallet: incoming_route_f_wallet.clone(), service_rate: rate_pack_f.routing_service_rate, byte_rate: rate_pack_f.routing_byte_rate } @@ -4294,22 +4552,22 @@ mod tests { &ReportServicesConsumedMessage { timestamp: second_report_timestamp, exit: ExitServiceConsumed { - earning_wallet: incoming_route_g_wallet, + earning_wallet: incoming_route_d_wallet, payload_size: second_exit_size, - service_rate: rate_pack_g.exit_service_rate, - byte_rate: rate_pack_g.exit_byte_rate + service_rate: rate_pack_d.exit_service_rate, + byte_rate: rate_pack_d.exit_byte_rate }, routing_payload_size: routing_size, routing: vec![ RoutingServiceConsumed { - earning_wallet: incoming_route_h_wallet, - service_rate: rate_pack_h.routing_service_rate, - byte_rate: rate_pack_h.routing_byte_rate + earning_wallet: incoming_route_e_wallet, + service_rate: rate_pack_e.routing_service_rate, + byte_rate: rate_pack_e.routing_byte_rate }, RoutingServiceConsumed { - earning_wallet: incoming_route_i_wallet, - service_rate: rate_pack_i.routing_service_rate, - byte_rate: rate_pack_i.routing_byte_rate + earning_wallet: incoming_route_f_wallet, + service_rate: rate_pack_f.routing_service_rate, + byte_rate: rate_pack_f.routing_byte_rate } ] } @@ -4339,48 +4597,48 @@ mod tests { .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); subject.logger = Logger::new(test_name); - let mut dns_failure_retries_hash_map = HashMap::new(); let mut dns_fail_client_payload = make_request_payload(111, cryptde); dns_fail_client_payload.stream_key = stream_key; - dns_failure_retries_hash_map.insert( - stream_key, - DNSFailureRetry { - unsuccessful_request: dns_fail_client_payload, - retries_left: 3, - }, - ); - subject.dns_failure_retries = dns_failure_retries_hash_map; let incoming_route_d_wallet = make_wallet("D Earning"); let incoming_route_e_wallet = make_wallet("E Earning"); let incoming_route_f_wallet = make_wallet("F Earning"); let rate_pack_d = rate_pack(101); let rate_pack_e = rate_pack(102); let rate_pack_f = rate_pack(103); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![ - ExpectedService::Exit( - irrelevant_public_key.clone(), - incoming_route_d_wallet.clone(), - rate_pack_d, - ), - ExpectedService::Routing( - irrelevant_public_key.clone(), - incoming_route_e_wallet.clone(), - rate_pack_e, - ), - ExpectedService::Routing( - irrelevant_public_key.clone(), - incoming_route_f_wallet.clone(), - rate_pack_f, + subject.stream_info.insert( + stream_key_clone.clone(), + StreamInfoBuilder::new() + .dns_failure_retry(DNSFailureRetry { + unsuccessful_request: dns_fail_client_payload, + retries_left: 3, + }) + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ + ExpectedService::Exit( + irrelevant_public_key.clone(), + incoming_route_d_wallet.clone(), + rate_pack_d, + ), + ExpectedService::Routing( + irrelevant_public_key.clone(), + incoming_route_e_wallet.clone(), + rate_pack_e, + ), + ExpectedService::Routing( + irrelevant_public_key.clone(), + incoming_route_f_wallet.clone(), + rate_pack_f, + ), + ExpectedService::Nothing, + ], ), - ExpectedService::Nothing, - ], - protocol: ProxyProtocol::TLS, - hostname_opt: None, - }, + host: Host::new("booga.com", TLS_PORT), + }) + .protocol(ProxyProtocol::TLS) + .build(), ); let subject_addr: Addr = subject.start(); let first_client_response_payload = ClientResponsePayload_0v1 { @@ -4395,7 +4653,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), first_client_response_payload.into(), 0, ); @@ -4407,14 +4665,16 @@ mod tests { subject_addr .try_send(AssertionsMessage { assertions: Box::new(move |proxy_server: &mut ProxyServer| { - let retry_opt = proxy_server.dns_failure_retries.get(&stream_key); - assert_eq!(retry_opt, None); + let retry_opt = &proxy_server + .stream_info(&stream_key_clone) + .unwrap() + .dns_failure_retry_opt; + assert!(retry_opt.is_none()); }), }) .unwrap(); System::current().stop(); system.run(); - TestLogHandler::new().exists_log_containing(&format!("DEBUG: {test_name}: Successful attempt of DNS resolution, removing DNS retry entry for stream key: {stream_key_clone}")); } #[test] @@ -4438,25 +4698,30 @@ mod tests { let incoming_route_e_wallet = make_wallet("E Earning"); let rate_pack_d = rate_pack(101); let rate_pack_e = rate_pack(102); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![ - ExpectedService::Exit( - irrelevant_public_key.clone(), - incoming_route_d_wallet.clone(), - rate_pack_d, - ), - ExpectedService::Routing( - irrelevant_public_key.clone(), - incoming_route_e_wallet.clone(), - rate_pack_e, + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ + ExpectedService::Exit( + irrelevant_public_key.clone(), + incoming_route_d_wallet.clone(), + rate_pack_d, + ), + ExpectedService::Routing( + irrelevant_public_key.clone(), + incoming_route_e_wallet.clone(), + rate_pack_e, + ), + ], ), - ], - protocol: ProxyProtocol::TLS, - hostname_opt: None, - }, + host: Host::new("booga.com", TLS_PORT), + }) + .protocol(ProxyProtocol::TLS) + .build(), ); let subject_addr: Addr = subject.start(); let client_response_payload = ClientResponsePayload_0v1 { @@ -4472,7 +4737,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), client_response_payload.into(), 0, ); @@ -4534,46 +4799,47 @@ mod tests { let stream_key = StreamKey::make_meaningless_stream_key(); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); - let mut dns_failure_retries_hash_map = HashMap::new(); let client_payload = make_request_payload(111, cryptde); - dns_failure_retries_hash_map.insert( - stream_key, - DNSFailureRetry { - unsuccessful_request: client_payload, - retries_left: 0, - }, - ); - subject.dns_failure_retries = dns_failure_retries_hash_map; + let exit_public_key = PublicKey::from(&b"exit_key"[..]); + let exit_wallet = make_wallet("exit wallet"); subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); - let exit_public_key = PublicKey::from(&b"exit_key"[..]); - let exit_wallet = make_wallet("exit wallet"); + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .dns_failure_retry(DNSFailureRetry { + unsuccessful_request: client_payload, + retries_left: 0, + }) + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ExpectedService::Exit( + exit_public_key.clone(), + exit_wallet, + rate_pack(10), + )], + ), + host: Host::new("server.com", HTTP_PORT), + }) + .protocol(ProxyProtocol::HTTP) + .build(), + ); let subject_addr: Addr = subject.start(); let dns_resolve_failure = DnsResolveFailure_0v1::new(stream_key); let expired_cores_package: ExpiredCoresPackage = ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), dns_resolve_failure.into(), 0, ); let peer_actors = peer_actors_builder().dispatcher(dispatcher_mock).build(); subject_addr.try_send(BindMessage { peer_actors }).unwrap(); - subject_addr - .try_send(AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![ExpectedService::Exit( - exit_public_key.clone(), - exit_wallet, - rate_pack(10), - )], - protocol: ProxyProtocol::HTTP, - hostname_opt: Some("server.com".to_string()), - }) - .unwrap(); subject_addr.try_send(expired_cores_package).unwrap(); System::current().stop(); @@ -4585,9 +4851,9 @@ mod tests { TransmitDataMsg { endpoint: Endpoint::Socket(socket_addr), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: ServerImpersonatorHttp {} - .dns_resolution_failure_response(Some("server.com".to_string()),), + .dns_resolution_failure_response("server.com".to_string()), }, *record ); @@ -4608,16 +4874,7 @@ mod tests { let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); let stream_key = StreamKey::make_meaningless_stream_key(); let irrelevant_public_key = PublicKey::from(&b"irrelevant"[..]); - let mut dns_failure_retries_hash_map = HashMap::new(); let client_payload = make_request_payload(111, cryptde); - dns_failure_retries_hash_map.insert( - stream_key, - DNSFailureRetry { - unsuccessful_request: client_payload, - retries_left: 0, - }, - ); - subject.dns_failure_retries = dns_failure_retries_hash_map; subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); @@ -4627,31 +4884,40 @@ mod tests { let rate_pack_d = rate_pack(101); let rate_pack_e = rate_pack(102); let rate_pack_f = rate_pack(103); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![ - ExpectedService::Exit( - irrelevant_public_key.clone(), - incoming_route_d_wallet.clone(), - rate_pack_d, - ), - ExpectedService::Routing( - irrelevant_public_key.clone(), - incoming_route_e_wallet.clone(), - rate_pack_e, - ), - ExpectedService::Routing( - irrelevant_public_key.clone(), - incoming_route_f_wallet.clone(), - rate_pack_f, + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .dns_failure_retry(DNSFailureRetry { + unsuccessful_request: client_payload, + retries_left: 0, + }) + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ + ExpectedService::Exit( + irrelevant_public_key.clone(), + incoming_route_d_wallet.clone(), + rate_pack_d, + ), + ExpectedService::Routing( + irrelevant_public_key.clone(), + incoming_route_e_wallet.clone(), + rate_pack_e, + ), + ExpectedService::Routing( + irrelevant_public_key.clone(), + incoming_route_f_wallet.clone(), + rate_pack_f, + ), + ExpectedService::Nothing, + ], ), - ExpectedService::Nothing, - ], - protocol: ProxyProtocol::TLS, - hostname_opt: Some("server.com".to_string()), - }, + host: Host::new("booga.com", TLS_PORT), + }) + .protocol(ProxyProtocol::TLS) + .build(), ); let subject_addr: Addr = subject.start(); let dns_resolve_failure_payload = DnsResolveFailure_0v1::new(stream_key); @@ -4659,7 +4925,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), dns_resolve_failure_payload.into(), 0, ); @@ -4724,34 +4990,34 @@ mod tests { ); let stream_key = StreamKey::make_meaningless_stream_key(); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); - let mut dns_failure_retries_hash_map = HashMap::new(); let client_payload = make_request_payload(111, cryptde); - dns_failure_retries_hash_map.insert( - stream_key, - DNSFailureRetry { - unsuccessful_request: client_payload, - retries_left: 0, - }, - ); subject.logger = Logger::new(test_name); - subject.dns_failure_retries = dns_failure_retries_hash_map; subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); let exit_public_key = PublicKey::from(&b"exit_key"[..]); let exit_wallet = make_wallet("exit wallet"); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![ExpectedService::Exit( - exit_public_key.clone(), - exit_wallet, - rate_pack(10), - )], - protocol: ProxyProtocol::HTTP, - hostname_opt: Some("server.com".to_string()), - }, + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .dns_failure_retry(DNSFailureRetry { + unsuccessful_request: client_payload, + retries_left: 0, + }) + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ExpectedService::Exit( + exit_public_key.clone(), + exit_wallet.clone(), + rate_pack(10), + )], + ), + host: Host::new("server.com", HTTP_PORT), + }) + .protocol(ProxyProtocol::HTTP) + .build(), ); let subject_addr: Addr = subject.start(); let dns_resolve_failure = DnsResolveFailure_0v1::new(stream_key); @@ -4759,7 +5025,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), dns_resolve_failure.into(), 0, ); @@ -4789,79 +5055,8 @@ mod tests { } #[test] - fn handle_dns_resolve_failure_does_not_send_message_to_neighborhood_when_server_is_not_specified( + fn handle_dns_resolve_failure_logs_when_stream_key_is_found_in_stream_info_but_not_keys_and_addrs( ) { - init_test_logging(); - let test_name = "handle_dns_resolve_failure_does_not_send_message_to_neighborhood_when_server_is_not_specified"; - let system = System::new(test_name); - let (neighborhood, _, neighborhood_recording_arc) = make_recorder(); - let cryptde = CRYPTDE_PAIR.main.as_ref(); - let mut subject = ProxyServer::new( - CRYPTDE_PAIR.clone(), - true, - Some(STANDARD_CONSUMING_WALLET_BALANCE), - false, - false, - ); - let stream_key = StreamKey::make_meaningless_stream_key(); - let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); - let mut dns_failure_retries_hash_map = HashMap::new(); - let client_payload = make_request_payload(111, cryptde); - dns_failure_retries_hash_map.insert( - stream_key, - DNSFailureRetry { - unsuccessful_request: client_payload, - retries_left: 0, - }, - ); - subject.logger = Logger::new(test_name); - subject.dns_failure_retries = dns_failure_retries_hash_map; - subject - .keys_and_addrs - .insert(stream_key.clone(), socket_addr); - let exit_public_key = PublicKey::from(&b"exit_key"[..]); - let exit_wallet = make_wallet("exit wallet"); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![ExpectedService::Exit( - exit_public_key.clone(), - exit_wallet, - rate_pack(10), - )], - protocol: ProxyProtocol::HTTP, - hostname_opt: None, - }, - ); - let subject_addr: Addr = subject.start(); - let dns_resolve_failure = DnsResolveFailure_0v1::new(stream_key); - let expired_cores_package: ExpiredCoresPackage = - ExpiredCoresPackage::new( - SocketAddr::from_str("1.2.3.4:1234").unwrap(), - Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), - dns_resolve_failure.into(), - 0, - ); - let peer_actors = peer_actors_builder().neighborhood(neighborhood).build(); - - subject_addr.try_send(BindMessage { peer_actors }).unwrap(); - subject_addr.try_send(expired_cores_package).unwrap(); - - System::current().stop(); - system.run(); - let neighborhood_recording = neighborhood_recording_arc.lock().unwrap(); - let record_opt = - neighborhood_recording.get_record_opt::(0); - assert_eq!(record_opt, None); - TestLogHandler::new().exists_log_containing(&format!( - "ERROR: {test_name}: Exit node {exit_public_key} complained of DNS failure, but was given no hostname to resolve." - )); - } - - #[test] - fn handle_dns_resolve_failure_logs_when_stream_key_be_gone_but_server_name_be_not() { init_test_logging(); let system = System::new("test"); let (neighborhood_mock, _, _) = make_recorder(); @@ -4874,43 +5069,39 @@ mod tests { false, ); let stream_key = StreamKey::make_meaningless_stream_key(); - let return_route_id = 1234; let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); - let mut dns_failure_retries_hash_map = HashMap::new(); let client_payload = make_request_payload(111, cryptde); - dns_failure_retries_hash_map.insert( - stream_key, - DNSFailureRetry { - unsuccessful_request: client_payload, - retries_left: 0, - }, - ); - subject.dns_failure_retries = dns_failure_retries_hash_map; - subject - .keys_and_addrs - .insert(stream_key.clone(), socket_addr.clone()); let exit_public_key = PublicKey::from(&b"exit_key"[..]); let exit_wallet = make_wallet("exit wallet"); - subject.route_ids_to_return_routes.insert( - return_route_id, - AddReturnRouteMessage { - return_route_id, - expected_services: vec![ExpectedService::Exit( - exit_public_key.clone(), - exit_wallet, - rate_pack(10), - )], - protocol: ProxyProtocol::HTTP, - hostname_opt: Some("server.com".to_string()), - }, + subject.stream_info.insert( + stream_key, + StreamInfoBuilder::new() + .dns_failure_retry(DNSFailureRetry { + unsuccessful_request: client_payload, + retries_left: 0, + }) + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ExpectedService::Exit( + exit_public_key.clone(), + exit_wallet.clone(), + rate_pack(10), + )], + ), + host: Host::new("server.com", HTTP_PORT), + }) + .protocol(ProxyProtocol::HTTP) + .build(), ); let subject_addr: Addr = subject.start(); let dns_resolve_failure = DnsResolveFailure_0v1::new(stream_key); let expired_cores_package: ExpiredCoresPackage = ExpiredCoresPackage::new( - SocketAddr::from_str("1.2.3.4:1234").unwrap(), + socket_addr, Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, return_route_id), + return_route(cryptde), dns_resolve_failure.into(), 0, ); @@ -4950,35 +5141,34 @@ mod tests { false, ); let stream_key = StreamKey::make_meaningless_stream_key(); - let return_route_id = 1234; let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); - let mut dns_failure_retries_hash_map = HashMap::new(); let client_payload = make_request_payload(111, cryptde); - dns_failure_retries_hash_map.insert( - stream_key, - DNSFailureRetry { - unsuccessful_request: client_payload, - retries_left: 0, - }, - ); - subject.dns_failure_retries = dns_failure_retries_hash_map; subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); let exit_public_key = PublicKey::from(&b"exit_key"[..]); let exit_wallet = make_wallet("exit wallet"); - subject.route_ids_to_return_routes.insert( - return_route_id, - AddReturnRouteMessage { - return_route_id, - expected_services: vec![ExpectedService::Exit( - exit_public_key.clone(), - exit_wallet, - rate_pack(10), - )], - protocol: ProxyProtocol::HTTP, - hostname_opt: None, - }, + subject.stream_info.insert( + stream_key, + StreamInfoBuilder::new() + .dns_failure_retry(DNSFailureRetry { + unsuccessful_request: client_payload, + retries_left: 0, + }) + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ExpectedService::Exit( + exit_public_key.clone(), + exit_wallet.clone(), + rate_pack(10), + )], + ), + host: Host::new("booga.com", HTTP_PORT), + }) + .protocol(ProxyProtocol::HTTP) + .build(), ); let subject_addr: Addr = subject.start(); let dns_resolve_failure = DnsResolveFailure_0v1::new(stream_key); @@ -4986,7 +5176,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, return_route_id), + return_route(cryptde), dns_resolve_failure.into(), 0, ); @@ -5003,12 +5193,10 @@ mod tests { System::current().stop(); system.run(); - TestLogHandler::new().exists_log_containing( - &format!( - "Discarding DnsResolveFailure message for from an unrecognized stream key {:?}", - stream_key - ) - ); + TestLogHandler::new().exists_log_containing(&format!( + "Discarding DnsResolveFailure message from an unrecognized stream key {:?}", + stream_key + )); } #[test] @@ -5033,47 +5221,38 @@ mod tests { subject.subs.as_mut().unwrap().dispatcher = peer_actors.dispatcher.from_dispatcher_client; let stream_key = StreamKey::make_meaningless_stream_key(); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); - let mut dns_failure_retries_hash_map = HashMap::new(); let client_payload = make_request_payload(111, cryptde); - dns_failure_retries_hash_map.insert( - stream_key, - DNSFailureRetry { - unsuccessful_request: client_payload, - retries_left: 0, - }, - ); - subject.dns_failure_retries = dns_failure_retries_hash_map; subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); - subject - .tunneled_hosts - .insert(stream_key.clone(), "tunneled host".to_string()); - subject.stream_key_routes.insert( - stream_key.clone(), - RouteQueryResponse { - route: Route { hops: vec![] }, - expected_services: ExpectedServices::OneWay(vec![]), - }, - ); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![ - make_exit_service_from_key(PublicKey::new(b"exit_node")), - ExpectedService::Nothing, - ], - protocol: ProxyProtocol::HTTP, - hostname_opt: None, - }, + subject.stream_info.insert( + stream_key, + StreamInfoBuilder::new() + .dns_failure_retry(DNSFailureRetry { + unsuccessful_request: client_payload, + retries_left: 0, + }) + .tunneled_host("tunneled host") + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ + make_exit_service_from_key(PublicKey::new(b"exit_node")), + ExpectedService::Nothing, + ], + ), + host: Host::new("booga.com", HTTP_PORT), + }) + .protocol(ProxyProtocol::HTTP) + .build(), ); let dns_resolve_failure = DnsResolveFailure_0v1::new(stream_key); let expired_cores_package: ExpiredCoresPackage = ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), dns_resolve_failure.into(), 0, ); @@ -5081,8 +5260,7 @@ mod tests { subject.handle_dns_resolve_failure(&expired_cores_package); assert!(subject.keys_and_addrs.is_empty()); - assert!(subject.stream_key_routes.is_empty()); - assert!(subject.tunneled_hosts.is_empty()); + assert!(subject.stream_info.get(&stream_key).is_none()); } #[test] @@ -5101,27 +5279,27 @@ mod tests { ); let stream_key = StreamKey::make_meaningless_stream_key(); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); - let mut dns_failure_retries_hash_map = HashMap::new(); let client_payload = make_request_payload(111, cryptde); - dns_failure_retries_hash_map.insert( - stream_key, - DNSFailureRetry { - unsuccessful_request: client_payload, - retries_left: 0, - }, - ); - subject.dns_failure_retries = dns_failure_retries_hash_map; subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![ExpectedService::Nothing, ExpectedService::Nothing], - protocol: ProxyProtocol::HTTP, - hostname_opt: Some("server.com".to_string()), - }, + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .dns_failure_retry(DNSFailureRetry { + unsuccessful_request: client_payload, + retries_left: 0, + }) + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ExpectedService::Nothing, ExpectedService::Nothing], + ), + host: Host::new("server.com", HTTP_PORT), + }) + .protocol(ProxyProtocol::HTTP) + .build(), ); let subject_addr: Addr = subject.start(); let dns_resolve_failure = DnsResolveFailure_0v1::new(stream_key); @@ -5129,7 +5307,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), dns_resolve_failure.into(), 0, ); @@ -5160,9 +5338,9 @@ mod tests { TransmitDataMsg { endpoint: Endpoint::Socket(socket_addr), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: ServerImpersonatorHttp {} - .dns_resolution_failure_response(Some("server.com".to_string()),), + .dns_resolution_failure_response("server.com".to_string()), }, *record ); @@ -5170,7 +5348,8 @@ mod tests { #[test] fn handle_dns_resolve_failure_sent_request_retry() { - let system = System::new("test"); + let test_name = "handle_dns_resolve_failure_sent_request_retry"; + let system = System::new(test_name); let resolve_message_params_arc = Arc::new(Mutex::new(vec![])); let (neighborhood_mock, _, _) = make_recorder(); let exit_public_key = PublicKey::from(&b"exit_key"[..]); @@ -5185,8 +5364,8 @@ mod tests { expected_services: ExpectedServices::RoundTrip( expected_services.clone(), expected_services.clone(), - 1234, ), + host: Host::new("booga.com", HTTP_PORT), }; let neighborhood_mock = neighborhood_mock .system_stop_conditions(match_lazily_every_type_id!(RouteQueryMessage)) @@ -5200,28 +5379,28 @@ mod tests { false, ); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); - let mut dns_failure_retries_hash_map = HashMap::new(); let client_payload = make_request_payload(111, cryptde); let stream_key = client_payload.stream_key; - dns_failure_retries_hash_map.insert( - stream_key, - DNSFailureRetry { - unsuccessful_request: client_payload.clone(), - retries_left: 3, - }, - ); - subject.dns_failure_retries = dns_failure_retries_hash_map; subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: expected_services.clone(), - protocol: ProxyProtocol::HTTP, - hostname_opt: Some("server.com".to_string()), - }, + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .dns_failure_retry(DNSFailureRetry { + unsuccessful_request: client_payload.clone(), + retries_left: 3, + }) + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + expected_services.clone(), + ), + host: Host::new("server.com", HTTP_PORT), + }) + .protocol(ProxyProtocol::HTTP) + .build(), ); let message_resolver = RouteQueryResponseResolverMock::default() .resolve_message_params(&resolve_message_params_arc); @@ -5236,7 +5415,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), dns_resolve_failure.into(), 0, ); @@ -5250,7 +5429,12 @@ mod tests { subject_addr .try_send(AssertionsMessage { assertions: Box::new(move |proxy_server: &mut ProxyServer| { - let retry = proxy_server.dns_failure_retries.get(&stream_key).unwrap(); + let retry = proxy_server + .stream_info(&stream_key) + .unwrap() + .dns_failure_retry_opt + .as_ref() + .unwrap(); assert_eq!(retry.retries_left, 2); }), }) @@ -5275,10 +5459,10 @@ mod tests { } #[test] - fn handle_dns_resolve_failure_logs_error_when_there_is_no_entry_in_the_hashmap_for_the_stream_key( + fn handle_dns_resolve_failure_logs_error_when_there_is_no_dns_failure_retry_entry_for_the_stream_key( ) { init_test_logging(); - let test_name = "handle_dns_resolve_failure_logs_error_when_there_is_no_entry_in_the_hashmap_for_the_stream_key"; + let test_name = "handle_dns_resolve_failure_logs_error_when_there_is_no_dns_failure_retry_entry_for_the_stream_key"; let system = System::new(test_name); let exit_public_key = PublicKey::from(&b"exit_key"[..]); let exit_wallet = make_wallet("exit wallet"); @@ -5301,14 +5485,19 @@ mod tests { subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: expected_services.clone(), - protocol: ProxyProtocol::HTTP, - hostname_opt: Some("server.com".to_string()), - }, + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + expected_services.clone(), + ), + host: Host::new("server.com", HTTP_PORT), + }) + .protocol(ProxyProtocol::HTTP) + .build(), ); let subject_addr: Addr = subject.start(); let dns_resolve_failure = DnsResolveFailure_0v1::new(stream_key); @@ -5316,7 +5505,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), dns_resolve_failure.into(), 0, ); @@ -5329,8 +5518,7 @@ mod tests { system.run(); TestLogHandler::new().exists_log_containing(&format!( "ERROR: {test_name}: While \ - handling ExpiredCoresPackage: No entry found inside dns_failure_retries hashmap for \ - the stream_key: AAAAAAAAAAAAAAAAAAAAAAAAAAA" + handling ExpiredCoresPackage: No DNSFailureRetry entry found for the stream_key: {stream_key}" )); } @@ -5353,8 +5541,8 @@ mod tests { expected_services: ExpectedServices::RoundTrip( expected_services.clone(), expected_services.clone(), - 1234, ), + host: Host::new("booga.com", HTTP_PORT), }; let neighborhood_mock = neighborhood_mock .system_stop_conditions(match_lazily_every_type_id!( @@ -5373,29 +5561,29 @@ mod tests { ); subject.logger = Logger::new(test_name); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); - let mut dns_failure_retries_hash_map = HashMap::new(); let client_payload = make_request_payload(111, cryptde); let stream_key = client_payload.stream_key; let stream_key_clone = stream_key.clone(); - dns_failure_retries_hash_map.insert( - stream_key, - DNSFailureRetry { - unsuccessful_request: client_payload.clone(), - retries_left: 3, - }, - ); - subject.dns_failure_retries = dns_failure_retries_hash_map; subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: expected_services.clone(), - protocol: ProxyProtocol::HTTP, - hostname_opt: Some("server.com".to_string()), - }, + subject.stream_info.insert( + stream_key_clone.clone(), + StreamInfoBuilder::new() + .dns_failure_retry(DNSFailureRetry { + unsuccessful_request: client_payload, + retries_left: 3, + }) + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + expected_services.clone(), + ), + host: Host::new("server.com", HTTP_PORT), + }) + .protocol(ProxyProtocol::HTTP) + .build(), ); let message_resolver_factory = RouteQueryResponseResolverFactoryMock::default() .make_params(&make_params_arc) @@ -5412,7 +5600,7 @@ mod tests { ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), + return_route(cryptde), dns_resolve_failure.into(), 0, ); @@ -5436,9 +5624,7 @@ mod tests { .try_send(AssertionsMessage { assertions: Box::new(move |proxy_server: &mut ProxyServer| { assert_eq!(proxy_server.keys_and_addrs.a_to_b(&stream_key), None); - assert_eq!(proxy_server.stream_key_routes.get(&stream_key), None); - assert_eq!(proxy_server.tunneled_hosts.get(&stream_key), None); - assert_eq!(proxy_server.dns_failure_retries.get(&stream_key), None); + assert_eq!(proxy_server.stream_info.get(&stream_key).is_none(), true); }), }) .unwrap(); @@ -5467,15 +5653,20 @@ mod tests { subject .keys_and_addrs .insert(stream_key.clone(), socket_addr.clone()); - let remaining_route = return_route_with_id(cryptde, 4321); - subject.route_ids_to_return_routes.insert( - 4321, - AddReturnRouteMessage { - return_route_id: 4321, - expected_services: vec![ExpectedService::Nothing], - protocol: ProxyProtocol::HTTP, - hostname_opt: None, - }, + let remaining_route = return_route(cryptde); + subject.stream_info.insert( + stream_key.clone(), + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip( + vec![], + vec![ExpectedService::Nothing], + ), + host: Host::new("booga.com", HTTP_PORT), + }) + .protocol(ProxyProtocol::HTTP) + .build(), ); let subject_addr: Addr = subject.start(); @@ -5518,8 +5709,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(80), - sequence_number: Some(0), + reception_port_opt: Some(80), + sequence_number_opt: Some(0), last_data: false, is_clandestine: false, data: expected_data.clone(), @@ -5557,57 +5748,7 @@ mod tests { .accountant(accountant) .build(); let client_response_payload = ClientResponsePayload_0v1 { - stream_key, - sequenced_packet: SequencedPacket { - data: b"some data".to_vec(), - sequence_number: 4321, - last_data: false, - }, - }; - let expired_cores_package = ExpiredCoresPackage::new( - SocketAddr::from_str("1.2.3.4:1234").unwrap(), - Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), - client_response_payload, - 0, - ); - subject_addr.try_send(BindMessage { peer_actors }).unwrap(); - - subject_addr.try_send(expired_cores_package).unwrap(); - - System::current().stop(); - system.run(); - TestLogHandler::new().exists_log_containing("ERROR: ProxyServer: Can't report services consumed: received response with bogus return-route ID 1234 for client response. Ignoring"); - assert_eq!(dispatcher_recording_arc.lock().unwrap().len(), 0); - assert_eq!(accountant_recording_arc.lock().unwrap().len(), 0); - } - - #[test] - fn report_response_services_consumed_complains_and_drops_package_if_return_route_id_is_unreadable( - ) { - init_test_logging(); - let cryptde = CRYPTDE_PAIR.main.as_ref(); - let (dispatcher, _, dispatcher_recording_arc) = make_recorder(); - let (accountant, _, accountant_recording_arc) = make_recorder(); - let system = System::new("report_response_services_consumed_complains_and_drops_package_if_return_route_id_is_unreadable"); - let mut subject = ProxyServer::new( - CRYPTDE_PAIR.clone(), - true, - Some(STANDARD_CONSUMING_WALLET_BALANCE), - false, - false, - ); - let stream_key = StreamKey::make_meaningless_stream_key(); - subject - .keys_and_addrs - .insert(stream_key, SocketAddr::from_str("1.2.3.4:5678").unwrap()); - let subject_addr: Addr = subject.start(); - let peer_actors = peer_actors_builder() - .dispatcher(dispatcher) - .accountant(accountant) - .build(); - let client_response_payload = ClientResponsePayload_0v1 { - stream_key, + stream_key: stream_key.clone(), sequenced_packet: SequencedPacket { data: b"some data".to_vec(), sequence_number: 4321, @@ -5617,9 +5758,7 @@ mod tests { let expired_cores_package = ExpiredCoresPackage::new( SocketAddr::from_str("1.2.3.4:1234").unwrap(), Some(make_wallet("irrelevant")), - Route { - hops: vec![make_cover_hop(cryptde), CryptData::new(&[0])], - }, + return_route(cryptde), client_response_payload, 0, ); @@ -5629,74 +5768,11 @@ mod tests { System::current().stop(); system.run(); - TestLogHandler::new().exists_log_containing( - "ERROR: ProxyServer: Can't report services consumed: DecryptionError(InvalidKey(\"Could not decrypt with", - ); + TestLogHandler::new().exists_log_containing(format!("ERROR: ProxyServer: Can't pay for return services consumed: received response with unrecognized stream key {}. Ignoring", stream_key).as_str()); assert_eq!(dispatcher_recording_arc.lock().unwrap().len(), 0); assert_eq!(accountant_recording_arc.lock().unwrap().len(), 0); } - #[test] - fn return_route_ids_expire_when_instructed() { - init_test_logging(); - let cryptde = CRYPTDE_PAIR.main.as_ref(); - let stream_key = StreamKey::make_meaningless_stream_key(); - - let (tx, rx) = unbounded(); - thread::spawn(move || { - let system = System::new("report_response_services_consumed_complains_and_drops_package_if_return_route_id_does_not_exist"); - let mut subject = ProxyServer::new( - CRYPTDE_PAIR.clone(), - true, - Some(STANDARD_CONSUMING_WALLET_BALANCE), - false, - false, - ); - subject.route_ids_to_return_routes = TtlHashMap::new(Duration::from_millis(250)); - subject - .keys_and_addrs - .insert(stream_key, SocketAddr::from_str("1.2.3.4:5678").unwrap()); - subject.route_ids_to_return_routes.insert( - 1234, - AddReturnRouteMessage { - return_route_id: 1234, - expected_services: vec![], - protocol: ProxyProtocol::TLS, - hostname_opt: None, - }, - ); - let subject_addr: Addr = subject.start(); - let peer_actors = peer_actors_builder().build(); - subject_addr.try_send(BindMessage { peer_actors }).unwrap(); - tx.send(subject_addr).unwrap(); - - system.run(); - }); - - let subject_addr = rx.recv().unwrap(); - - thread::sleep(Duration::from_millis(300)); - - let client_response_payload = ClientResponsePayload_0v1 { - stream_key, - sequenced_packet: SequencedPacket { - data: b"some data".to_vec(), - sequence_number: 4321, - last_data: false, - }, - }; - let expired_cores_package = ExpiredCoresPackage::new( - SocketAddr::from_str("1.2.3.4:1234").unwrap(), - Some(make_wallet("irrelevant")), - return_route_with_id(cryptde, 1234), - client_response_payload, - 0, - ); - subject_addr.try_send(expired_cores_package).unwrap(); - - TestLogHandler::new().await_log_containing("ERROR: ProxyServer: Can't report services consumed: received response with bogus return-route ID 1234 for client response. Ignoring", 1000); - } - #[test] fn handle_stream_shutdown_msg_handles_unknown_peer_addr() { let mut subject = ProxyServer::new(CRYPTDE_PAIR.clone(), true, None, false, false); @@ -5705,16 +5781,17 @@ mod tests { subject .keys_and_addrs .insert(unaffected_stream_key, unaffected_socket_addr); - subject.stream_key_routes.insert( + subject.stream_info.insert( unaffected_stream_key, - RouteQueryResponse { - route: Route { hops: vec![] }, - expected_services: ExpectedServices::RoundTrip(vec![], vec![], 1234), - }, + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip(vec![], vec![]), + host: Host::new("booga.com", HTTP_PORT), + }) + .tunneled_host("blah") + .build(), ); - subject - .tunneled_hosts - .insert(unaffected_stream_key, "blah".to_string()); subject.handle_stream_shutdown_msg(StreamShutdownMsg { peer_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), @@ -5730,10 +5807,12 @@ mod tests { .keys_and_addrs .a_to_b(&unaffected_stream_key) .is_some()); + assert!(subject.stream_info.contains_key(&unaffected_stream_key)); assert!(subject - .stream_key_routes - .contains_key(&unaffected_stream_key)); - assert!(subject.tunneled_hosts.contains_key(&unaffected_stream_key)); + .stream_info(&unaffected_stream_key) + .unwrap() + .tunneled_host_opt + .is_some()); } #[test] @@ -5757,12 +5836,16 @@ mod tests { subject .keys_and_addrs .insert(affected_stream_key, affected_socket_addr); - subject.stream_key_routes.insert( + subject.stream_info.insert( unaffected_stream_key, - RouteQueryResponse { - route: Route { hops: vec![] }, - expected_services: ExpectedServices::RoundTrip(vec![], vec![], 1234), - }, + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip(vec![], vec![]), + host: Host::new("gooba.com", HTTP_PORT), + }) + .tunneled_host("blah") + .build(), ); let affected_route = Route::round_trip( RouteSegment::new( @@ -5781,7 +5864,6 @@ mod tests { ), CRYPTDE_PAIR.main.as_ref(), Some(make_paying_wallet(b"consuming")), - 1234, Some(TEST_DEFAULT_CHAIN.rec().contract), ) .unwrap(); @@ -5790,23 +5872,20 @@ mod tests { make_paying_wallet(b"1234"), DEFAULT_RATE_PACK, )]; - subject.stream_key_routes.insert( + subject.stream_info.insert( affected_stream_key, - RouteQueryResponse { - route: affected_route.clone(), - expected_services: ExpectedServices::RoundTrip( - affected_expected_services, - vec![], - 1234, - ), - }, + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: affected_route.clone(), + expected_services: ExpectedServices::RoundTrip( + affected_expected_services, + vec![], + ), + host: Host::new("gooba.com", TLS_PORT), + }) + .tunneled_host("tunneled.com") + .build(), ); - subject - .tunneled_hosts - .insert(unaffected_stream_key, "blah".to_string()); - subject - .tunneled_hosts - .insert(affected_stream_key, "tunneled.com".to_string()); let subject_addr = subject.start(); let (hopper, _, hopper_recording_arc) = make_recorder(); let (proxy_server, _, proxy_server_recording_arc) = make_recorder(); @@ -5840,8 +5919,8 @@ mod tests { ClientRequestPayload_0v1 { stream_key: affected_stream_key, sequenced_packet: SequencedPacket::new(vec![], 1234, true), - target_hostname: Some(String::from("tunneled.com")), - target_port: 443, + target_hostname: String::from("tunneled.com"), + target_port: TLS_PORT, protocol: ProxyProtocol::TLS, originator_public_key: CRYPTDE_PAIR.alias.as_ref().public_key().clone(), } @@ -5887,13 +5966,17 @@ mod tests { subject .keys_and_addrs .insert(affected_stream_key, affected_socket_addr); - subject.stream_key_routes.insert( + subject.stream_info.insert( unaffected_stream_key, - RouteQueryResponse { - route: Route { hops: vec![] }, - expected_services: ExpectedServices::RoundTrip(vec![], vec![], 1234), - }, + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip(vec![], vec![]), + host: Host::new("booga.com", HTTP_PORT), + }) + .build(), ); + subject.next_return_route_id = Cell::new(1234); let affected_route = Route::round_trip( RouteSegment::new( vec![ @@ -5911,7 +5994,6 @@ mod tests { ), CRYPTDE_PAIR.main.as_ref(), Some(make_paying_wallet(b"consuming")), - 1234, Some(TEST_DEFAULT_CHAIN.rec().contract), ) .unwrap(); @@ -5920,16 +6002,18 @@ mod tests { make_paying_wallet(b"1234"), DEFAULT_RATE_PACK, )]; - subject.stream_key_routes.insert( + subject.stream_info.insert( affected_stream_key, - RouteQueryResponse { - route: affected_route.clone(), - expected_services: ExpectedServices::RoundTrip( - affected_expected_services, - vec![], - 1234, - ), - }, + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: affected_route.clone(), + expected_services: ExpectedServices::RoundTrip( + affected_expected_services, + vec![], + ), + host: Host::new("booga.com", HTTP_PORT), + }) + .build(), ); subject.logger = Logger::new(test_name); let subject_addr = subject.start(); @@ -5965,7 +6049,7 @@ mod tests { ClientRequestPayload_0v1 { stream_key: affected_stream_key, sequenced_packet: SequencedPacket::new(vec![], 1234, true), - target_hostname: None, + target_hostname: "booga.com".to_string(), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: CRYPTDE_PAIR.alias.as_ref().public_key().clone(), @@ -6004,6 +6088,9 @@ mod tests { let socket_addr = SocketAddr::from_str("3.4.5.6:7777").unwrap(); let stream_key = StreamKey::make_meaningful_stream_key("All Things Must Pass"); subject.keys_and_addrs.insert(stream_key, socket_addr); + subject + .stream_info + .insert(stream_key.clone(), StreamInfoBuilder::new().build()); let msg = StreamShutdownMsg { peer_addr: socket_addr, stream_type: RemovedStreamType::NonClandestine(NonClandestineAttributes { @@ -6030,16 +6117,17 @@ mod tests { let socket_addr = SocketAddr::from_str("3.4.5.6:7890").unwrap(); let stream_key = StreamKey::make_meaningful_stream_key("All Things Must Pass"); subject.keys_and_addrs.insert(stream_key, socket_addr); - subject.stream_key_routes.insert( + subject.stream_info.insert( stream_key, - RouteQueryResponse { - route: Route { hops: vec![] }, - expected_services: ExpectedServices::RoundTrip(vec![], vec![], 0), - }, + StreamInfoBuilder::new() + .route(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip(vec![], vec![]), + host: Host::new("booga.com", HTTP_PORT), + }) + .tunneled_host("blah") + .build(), ); - subject - .tunneled_hosts - .insert(stream_key, "blah".to_string()); let msg = StreamShutdownMsg { peer_addr: socket_addr, stream_type: RemovedStreamType::NonClandestine(NonClandestineAttributes { @@ -6055,14 +6143,13 @@ mod tests { let after = SystemTime::now(); let handle_normal_client_data = help_to_handle_normal_client_data_params_arc.lock().unwrap(); - let (inbound_client_data_msg, retire_stream_key) = &handle_normal_client_data[0]; + let inbound_client_data_msg = &handle_normal_client_data[0]; assert_eq!(inbound_client_data_msg.client_addr, socket_addr); assert_eq!(inbound_client_data_msg.data, Vec::::new()); assert_eq!(inbound_client_data_msg.last_data, true); assert_eq!(inbound_client_data_msg.is_clandestine, false); let actual_timestamp = inbound_client_data_msg.timestamp; assert!(before <= actual_timestamp && actual_timestamp <= after); - assert_eq!(*retire_stream_key, true) } #[test] @@ -6072,18 +6159,15 @@ mod tests { let inbound_client_data_msg = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:4578").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: true, is_clandestine: false, - sequence_number: Some(123), + sequence_number_opt: Some(123), data: vec![], }; - let result = IBCDHelperReal::new().handle_normal_client_data( - &mut proxy_server, - inbound_client_data_msg, - true, - ); + let result = IBCDHelperReal::new() + .handle_normal_client_data(&mut proxy_server, inbound_client_data_msg); assert_eq!( result, @@ -6102,6 +6186,7 @@ mod tests { let args = TransmitToHopperArgs { main_cryptde: cryptde.dup(), payload, + return_route_id: 8888, client_addr: SocketAddr::from_str("1.2.3.4:1234").unwrap(), timestamp: SystemTime::now(), is_decentralized: false, @@ -6111,16 +6196,10 @@ mod tests { accountant_sub: recipient!(&addr, ReportServicesConsumedMessage), retire_stream_key_sub_opt: None, }; - let add_return_route_sub = recipient!(&addr, AddReturnRouteMessage); let subject = RouteQueryResponseResolverReal {}; let system = System::new("resolve_message_handles_mailbox_error_from_neighborhood"); - subject.resolve_message( - args, - add_return_route_sub, - proxy_server_sub, - Err(MailboxError::Timeout), - ); + subject.resolve_message(args, proxy_server_sub, Err(MailboxError::Timeout)); System::current().stop(); system.run(); @@ -6138,22 +6217,63 @@ mod tests { #[derive(Default)] struct ClientRequestPayloadFactoryMock { + make_params: Arc< + Mutex< + Vec<( + InboundClientData, + StreamKey, + Option, + Box, + Logger, + )>, + >, + >, make_results: RefCell>>, } impl ClientRequestPayloadFactory for ClientRequestPayloadFactoryMock { fn make( &self, - _ibcd: &InboundClientData, - _stream_key: StreamKey, - _cryptde: &dyn CryptDE, - _logger: &Logger, + ibcd: &InboundClientData, + stream_key: StreamKey, + host_opt: Option, + cryptde: &dyn CryptDE, + logger: &Logger, ) -> Option { + self.make_params.lock().unwrap().push(( + ibcd.clone(), + stream_key, + host_opt, + cryptde.dup(), + logger.clone(), + )); self.make_results.borrow_mut().remove(0) } } impl ClientRequestPayloadFactoryMock { + fn new() -> Self { + Self::default() + } + + fn make_params( + mut self, + params: &Arc< + Mutex< + Vec<( + InboundClientData, + StreamKey, + Option, + Box, + Logger, + )>, + >, + >, + ) -> Self { + self.make_params = params.clone(); + self + } + fn make_result(self, result: Option) -> Self { self.make_results.borrow_mut().push(result); self @@ -6175,18 +6295,15 @@ mod tests { let inbound_client_data_msg = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:4578").unwrap(), - reception_port: Some(568), + reception_port_opt: Some(568), last_data: true, is_clandestine: false, - sequence_number: Some(123), + sequence_number_opt: Some(123), data: vec![], }; - let result = IBCDHelperReal::new().handle_normal_client_data( - &mut proxy_server, - inbound_client_data_msg, - true, - ); + let result = IBCDHelperReal::new() + .handle_normal_client_data(&mut proxy_server, inbound_client_data_msg); assert_eq!( result, @@ -6196,7 +6313,7 @@ mod tests { #[test] fn new_http_request_creates_new_entry_inside_dns_retries_hashmap() { - let alias_cryptde = CRYPTDE_PAIR.alias.as_ref(); + let test_name = "new_http_request_creates_new_entry_inside_dns_retries_hashmap"; let http_request = b"GET /index.html HTTP/1.1\r\nHost: nowhere.com\r\n\r\n"; let (neighborhood_mock, _, _) = make_recorder(); let destination_key = PublicKey::from(&b"our destination"[..]); @@ -6205,8 +6322,8 @@ mod tests { expected_services: ExpectedServices::RoundTrip( vec![make_exit_service_from_key(destination_key.clone())], vec![], - 1234, ), + host: Host::new("booga.com", HTTP_PORT), })); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); let stream_key = StreamKey::make_meaningless_stream_key(); @@ -6214,8 +6331,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data.clone(), @@ -6224,14 +6341,13 @@ mod tests { .make( &msg_from_dispatcher, stream_key.clone(), - alias_cryptde, + None, + CRYPTDE_PAIR.alias.as_ref(), &Logger::new("test"), ) .unwrap(); let stream_key_factory = StreamKeyFactoryMock::new().make_result(stream_key.clone()); - let system = System::new( - "proxy_server_receives_http_request_from_dispatcher_then_sends_cores_package_to_hopper", - ); + let system = System::new(test_name); let mut subject = ProxyServer::new( CRYPTDE_PAIR.clone(), true, @@ -6251,7 +6367,12 @@ mod tests { subject_addr .try_send(AssertionsMessage { assertions: Box::new(move |proxy_server: &mut ProxyServer| { - let dns_retry = proxy_server.dns_failure_retries.get(&stream_key).unwrap(); + let dns_retry = proxy_server + .stream_info(&stream_key) + .unwrap() + .dns_failure_retry_opt + .as_ref() + .unwrap(); assert_eq!(dns_retry.retries_left, 3); assert_eq!(dns_retry.unsuccessful_request, expected_payload); }), @@ -6263,6 +6384,8 @@ mod tests { #[test] fn new_http_request_creates_new_exhausted_entry_inside_dns_retries_hashmap_zero_hop() { + let test_name = + "new_http_request_creates_new_exhausted_entry_inside_dns_retries_hashmap_zero_hop"; let http_request = b"GET /index.html HTTP/1.1\r\nHost: nowhere.com\r\n\r\n"; let (neighborhood_mock, _, _) = make_recorder(); let destination_key = PublicKey::from(&b"our destination"[..]); @@ -6271,8 +6394,8 @@ mod tests { expected_services: ExpectedServices::RoundTrip( vec![make_exit_service_from_key(destination_key.clone())], vec![], - 1234, ), + host: Host::new("booga.com", HTTP_PORT), })); let socket_addr = SocketAddr::from_str("1.2.3.4:5678").unwrap(); let stream_key = StreamKey::make_meaningless_stream_key(); @@ -6280,8 +6403,8 @@ mod tests { let msg_from_dispatcher = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr.clone(), - reception_port: Some(HTTP_PORT), - sequence_number: Some(0), + reception_port_opt: Some(HTTP_PORT), + sequence_number_opt: Some(0), last_data: true, is_clandestine: false, data: expected_data.clone(), @@ -6290,14 +6413,13 @@ mod tests { .make( &msg_from_dispatcher, stream_key.clone(), + None, CRYPTDE_PAIR.alias.as_ref(), &Logger::new("test"), ) .unwrap(); let stream_key_factory = StreamKeyFactoryMock::new().make_result(stream_key.clone()); - let system = System::new( - "new_http_request_creates_new_exhausted_entry_inside_dns_retries_hashmap_zero_hop", - ); + let system = System::new(test_name); let mut subject = ProxyServer::new( CRYPTDE_PAIR.clone(), false, @@ -6317,7 +6439,12 @@ mod tests { subject_addr .try_send(AssertionsMessage { assertions: Box::new(move |proxy_server: &mut ProxyServer| { - let dns_retry = proxy_server.dns_failure_retries.get(&stream_key).unwrap(); + let dns_retry = proxy_server + .stream_info(&stream_key) + .unwrap() + .dns_failure_retry_opt + .as_ref() + .unwrap(); assert_eq!(dns_retry.retries_left, 0); assert_eq!(dns_retry.unsuccessful_request, expected_payload); }), @@ -6407,18 +6534,15 @@ mod tests { let inbound_client_data_msg = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.2.3.4:4578").unwrap(), - reception_port: Some(80), + reception_port_opt: Some(80), last_data: true, is_clandestine: false, - sequence_number: Some(123), + sequence_number_opt: Some(123), data: expected_data, }; - let result = IBCDHelperReal::new().handle_normal_client_data( - &mut proxy_server, - inbound_client_data_msg, - true, - ); + let result = IBCDHelperReal::new() + .handle_normal_client_data(&mut proxy_server, inbound_client_data_msg); assert_eq!( result, @@ -6426,6 +6550,78 @@ mod tests { ); } + #[test] + fn make_payload_passes_no_hostname_if_none_is_known() { + let mut subject = ProxyServer::new(CRYPTDE_PAIR.clone(), true, Some(58), false, false); + let make_params_arc = Arc::new(Mutex::new(vec![])); + let client_request_payload_factory = ClientRequestPayloadFactoryMock::new() + .make_params(&make_params_arc) + .make_result(None); + subject.client_request_payload_factory = Box::new(client_request_payload_factory); + let stream_key = StreamKey::make_meaningless_stream_key(); + // Do not create an entry in subject.stream_info for stream_key, so that no hostname is known + + let _ = subject.make_payload( + InboundClientData { + // irrelevant + timestamp: SystemTime::now(), + client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), + reception_port_opt: Some(HTTP_PORT), + last_data: false, + is_clandestine: false, + sequence_number_opt: Some(123), + data: vec![], + }, + &stream_key, + ); + + let (_ibcd, _sk, hostname_opt, _cryptde, _logger) = &make_params_arc.lock().unwrap()[0]; + assert_eq!(hostname_opt, &None); + } + + #[test] + fn make_payload_passes_hostname_if_known() { + let mut subject = ProxyServer::new(CRYPTDE_PAIR.clone(), true, Some(58), false, false); + let make_params_arc = Arc::new(Mutex::new(vec![])); + let client_request_payload_factory = ClientRequestPayloadFactoryMock::new() + .make_params(&make_params_arc) + .make_result(None); // Don't care about return value, only parameters + subject.client_request_payload_factory = Box::new(client_request_payload_factory); + let stream_key = StreamKey::make_meaningless_stream_key(); + let si_host = Host::new("knownhostname.com", 2345); + subject.stream_info.insert( + stream_key.clone(), + StreamInfo { + tunneled_host_opt: None, + dns_failure_retry_opt: None, + route_opt: Some(RouteQueryResponse { + route: Route { hops: vec![] }, + expected_services: ExpectedServices::RoundTrip(vec![], vec![]), + host: Host::new(&si_host.name, 2345), + }), + protocol_opt: None, + time_to_live_opt: None, + }, + ); + + let _ = subject.make_payload( + InboundClientData { + // irrelevant + timestamp: SystemTime::now(), + client_addr: SocketAddr::from_str("1.2.3.4:5678").unwrap(), + reception_port_opt: Some(HTTP_PORT), + last_data: false, + is_clandestine: false, + sequence_number_opt: Some(123), + data: vec![], + }, + &stream_key, + ); + + let (_ibcd, _sk, host_opt, _cryptde, _logger) = &make_params_arc.lock().unwrap()[0]; + assert_eq!(host_opt, &Some(si_host)); + } + #[test] #[should_panic( expected = "ProxyServer should never get ShutdownStreamMsg about clandestine stream" @@ -6466,10 +6662,10 @@ mod tests { let ibcd = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr, - reception_port: Some(2222), + reception_port_opt: Some(2222), last_data: true, is_clandestine: false, - sequence_number: Some(333), + sequence_number_opt: Some(333), data: b"GET /index.html HTTP/1.1\r\nHost: header.com:3333\r\n\r\n".to_vec(), }; @@ -6490,10 +6686,10 @@ mod tests { let ibcd = InboundClientData { timestamp: SystemTime::now(), client_addr: socket_addr, - reception_port: Some(2222), + reception_port_opt: Some(2222), last_data: true, is_clandestine: false, - sequence_number: Some(333), + sequence_number_opt: Some(333), data: b"GET /index.html HTTP/1.1\r\nHost: header.com:4444\r\n\r\n".to_vec(), }; @@ -6506,6 +6702,30 @@ mod tests { ); } + fn make_server_com_client_hello() -> Vec { + [ + 0x16, // content_type: Handshake + 0x00, 0x00, 0x00, 0x00, // version, length: don't care + 0x01, // handshake_type: ClientHello + 0x00, 0x00, 0x00, 0x00, 0x00, // length, version: don't care + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, // random: don't care + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, // random: don't care + 0x00, // session_id_length + 0x00, 0x00, // cipher_suites_length + 0x00, // compression_methods_length + 0x00, 0x13, // extensions_length + 0x00, 0x00, // extension_type: server_name + 0x00, 0x0F, // extension_length + 0x00, 0x0D, // server_name_list_length + 0x00, // server_name_type + 0x00, 0x0A, // server_name_length + b's', b'e', b'r', b'v', b'e', b'r', b'.', b'c', b'o', b'm', // server_name + ] + .to_vec() + } + fn make_exit_service_from_key(public_key: PublicKey) -> ExpectedService { ExpectedService::Exit(public_key, make_wallet("exit wallet"), rate_pack(100)) } diff --git a/node/src/proxy_server/protocol_pack.rs b/node/src/proxy_server/protocol_pack.rs index 9697ce6a2..e5607bb08 100644 --- a/node/src/proxy_server/protocol_pack.rs +++ b/node/src/proxy_server/protocol_pack.rs @@ -3,29 +3,16 @@ use crate::proxy_server::http_protocol_pack::HttpProtocolPack; use crate::proxy_server::tls_protocol_pack::TlsProtocolPack; use crate::sub_lib::cryptde::PlainData; use crate::sub_lib::dispatcher::InboundClientData; +use crate::sub_lib::host::Host; use crate::sub_lib::proxy_server::ProxyProtocol; use masq_lib::constants::{HTTP_PORT, TLS_PORT}; -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Host { - pub name: String, - pub port: u16, -} - -impl Host { - pub fn new(name: &str, port: u16) -> Host { - Host { - name: name.to_string(), - port, - } - } -} - pub trait ProtocolPack: Send + Sync { fn proxy_protocol(&self) -> ProxyProtocol; fn standard_port(&self) -> u16; fn find_host(&self, data: &PlainData) -> Option; fn server_impersonator(&self) -> Box; + fn describe_packet(&self, data: &PlainData) -> String; } pub fn from_protocol(protocol: ProxyProtocol) -> Box { @@ -44,7 +31,7 @@ pub fn from_standard_port(standard_port: u16) -> Option> { } pub fn from_ibcd(ibcd: &InboundClientData) -> Result, String> { - let origin_port = match ibcd.reception_port { + let origin_port = match ibcd.reception_port_opt { None => { return Err(format!( "No origin port specified with {}-byte non-clandestine packet: {:?}", @@ -67,6 +54,6 @@ pub fn from_ibcd(ibcd: &InboundClientData) -> Result, Stri pub trait ServerImpersonator { fn route_query_failure_response(&self, server_name: &str) -> Vec; - fn dns_resolution_failure_response(&self, server_name_opt: Option) -> Vec; + fn dns_resolution_failure_response(&self, server_name: String) -> Vec; fn consuming_wallet_absent(&self) -> Vec; } diff --git a/node/src/proxy_server/server_impersonator_http.rs b/node/src/proxy_server/server_impersonator_http.rs index 554741aae..9de02301d 100644 --- a/node/src/proxy_server/server_impersonator_http.rs +++ b/node/src/proxy_server/server_impersonator_http.rs @@ -19,11 +19,9 @@ impl ServerImpersonator for ServerImpersonatorHttp { ) } - fn dns_resolution_failure_response(&self, server_name_opt: Option) -> Vec { - let (server_name, quoted_server_name) = match &server_name_opt { - Some(name) => (name.clone(), format!("\"{}\"", name)), - None => ("".to_string(), "".to_string()), - }; + fn dns_resolution_failure_response(&self, server_name: String) -> Vec { + let (server_name, quoted_server_name) = + (server_name.clone(), format!("\"{}\"", server_name)); ServerImpersonatorHttp::make_error_response( 503, "DNS Resolution Problem", @@ -197,7 +195,7 @@ mod tests { fn dns_resolution_failure_response_with_server_name_produces_expected_error_page() { let subject = ServerImpersonatorHttp {}; - let result = subject.dns_resolution_failure_response(Some("server.com".to_string())); + let result = subject.dns_resolution_failure_response("server.com".to_string()); let expected = ServerImpersonatorHttp::make_error_response( 503, @@ -208,21 +206,6 @@ mod tests { assert_eq!(expected, result); } - #[test] - fn dns_resolution_failure_response_without_server_name_produces_expected_error_page() { - let subject = ServerImpersonatorHttp {}; - - let result = subject.dns_resolution_failure_response(None); - - let expected = ServerImpersonatorHttp::make_error_response( - 503, - "DNS Resolution Problem", - "Exit Nodes couldn't resolve ", - "DNS Failure, We have tried multiple Exit Nodes and all have failed to resolve this address ", - ); - assert_eq!(expected, result); - } - #[test] fn consuming_wallet_absent_response_produces_expected_error_page() { let subject = ServerImpersonatorHttp {}; diff --git a/node/src/proxy_server/server_impersonator_tls.rs b/node/src/proxy_server/server_impersonator_tls.rs index 7a193a269..3312be656 100644 --- a/node/src/proxy_server/server_impersonator_tls.rs +++ b/node/src/proxy_server/server_impersonator_tls.rs @@ -8,7 +8,7 @@ impl ServerImpersonator for ServerImpersonatorTls { Vec::from(&TLS_INTERNAL_ERROR_ALERT[..]) } - fn dns_resolution_failure_response(&self, _server_name: Option) -> Vec { + fn dns_resolution_failure_response(&self, _server_name: String) -> Vec { Vec::from(&TLS_UNRECOGNIZED_NAME_ALERT[..]) } @@ -75,7 +75,7 @@ mod tests { fn dns_resolution_failure_response_produces_unrecognized_name_alert() { let subject = ServerImpersonatorTls {}; - let result = subject.dns_resolution_failure_response(None); + let result = subject.dns_resolution_failure_response("booga.com".to_string()); assert_eq!(Vec::from(&TLS_UNRECOGNIZED_NAME_ALERT[..]), result); } diff --git a/node/src/proxy_server/tls_protocol_pack.rs b/node/src/proxy_server/tls_protocol_pack.rs index d167ef0fa..bb81f8543 100644 --- a/node/src/proxy_server/tls_protocol_pack.rs +++ b/node/src/proxy_server/tls_protocol_pack.rs @@ -1,11 +1,13 @@ // Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. -use crate::proxy_server::protocol_pack::{Host, ProtocolPack, ServerImpersonator}; +use crate::proxy_server::protocol_pack::{ProtocolPack, ServerImpersonator}; use crate::proxy_server::server_impersonator_tls::ServerImpersonatorTls; use crate::sub_lib::binary_traverser::BinaryTraverser; use crate::sub_lib::cryptde::PlainData; +use crate::sub_lib::host::Host; use crate::sub_lib::proxy_server::ProxyProtocol; use masq_lib::constants::TLS_PORT; +#[derive(Clone, Copy)] pub struct TlsProtocolPack {} impl ProtocolPack for TlsProtocolPack { @@ -37,6 +39,21 @@ impl ProtocolPack for TlsProtocolPack { fn server_impersonator(&self) -> Box { Box::new(ServerImpersonatorTls {}) } + + fn describe_packet(&self, data: &PlainData) -> String { + match data.get_u8(0) { + Some(0x16u8) => self.describe_handshake(data), + Some(0x14u8) => self.describe_cipher_spec(data), + Some(0x15u8) => self.describe_alert(data), + Some(0x17u8) => self.describe_application_data(data), + Some(opcode) => format!( + "{}-byte packet of unrecognized type 0x{:02X}", + data.len(), + opcode + ), + None => format!("Incomplete {}-byte TLS packet", data.len()), + } + } } impl TlsProtocolPack { @@ -99,6 +116,89 @@ impl TlsProtocolPack { Err(_) => Err(()), } } + + fn describe_handshake(&self, data: &PlainData) -> String { + match data.get_u8(5) { + Some(0x00u8) => "HelloRequest".to_string(), + Some(0x01u8) => self.describe_client_hello(data), + Some(0x02u8) => "ServerHello".to_string(), + Some(0x03u8) => "HelloVerifyRequest".to_string(), + Some(0x04u8) => "NewSessionTicket".to_string(), + Some(0x05u8) => "EndOfEarlyData".to_string(), + Some(0x06u8) => "HelloRetryRequest".to_string(), + Some(0x0Bu8) => "Certificate".to_string(), + Some(0x0Cu8) => "ServerKeyExchange".to_string(), + Some(0x0Du8) => "CertificateRequest".to_string(), + Some(0x0Eu8) => "ServerHelloDone".to_string(), + Some(0x0Fu8) => "CertificateVerify".to_string(), + Some(0x10u8) => "ClientKeyExchange".to_string(), + Some(0x14u8) => "Finished".to_string(), + Some(opcode) => format!("Unrecognized Handshake 0x{:02X}", opcode), + None => format!("Incomplete {}-byte Handshake packet", data.len()), + } + } + + fn describe_client_hello(&self, data: &PlainData) -> String { + match self.find_host(data) { + Some(host) => format!("ClientHello with SNI '{}'", host.name), + None => "ClientHello with no SNI extension".to_string(), + } + } + + fn describe_cipher_spec(&self, data: &PlainData) -> String { + match data.get_u8(5) { + Some(0x01u8) => "ChangeCipherSpec".to_string(), + Some(opcode) => format!("Unrecognized ChangeCipherSpec 0x{:02X}", opcode), + None => format!("Incomplete {}-byte ChangeCipherSpec packet", data.len()), + } + } + + fn describe_alert(&self, data: &PlainData) -> String { + let level = match data.get_u8(5) { + Some(0x01u8) => "Warning".to_string(), + Some(0x02u8) => "Fatal".to_string(), + Some(opcode) => format!("Unrecognized Alert Level 0x{:02X}", opcode), + None => return format!("Incomplete {}-byte Alert packet", data.len()), + }; + let description = match data.get_u8(6) { + Some(0x00) => "CloseNotify".to_string(), + Some(0x01) => "Unrecognized Alert Description 0x01".to_string(), + Some(0x0A) => "UnexpectedMessage".to_string(), + Some(0x14) => "BadRecordMAC".to_string(), + Some(0x15) => "DecryptionFailed".to_string(), + Some(0x16) => "RecordOverflow".to_string(), + Some(0x1E) => "DecompressionFailure".to_string(), + Some(0x28) => "HandshakeFailure".to_string(), + Some(0x29) => "NoCertificate".to_string(), + Some(0x2A) => "BadCertificate".to_string(), + Some(0x2B) => "UnsupportedCertificate".to_string(), + Some(0x2C) => "CertificateRevoked".to_string(), + Some(0x2D) => "CertificateExpired".to_string(), + Some(0x2E) => "CertificateUnknown".to_string(), + Some(0x2F) => "IllegalParameter".to_string(), + Some(0x30) => "UnknownCA".to_string(), + Some(0x31) => "AccessDenied".to_string(), + Some(0x32) => "DecodeError".to_string(), + Some(0x33) => "DecryptError".to_string(), + Some(0x3C) => "ExportRestriction".to_string(), + Some(0x46) => "ProtocolVersion".to_string(), + Some(0x47) => "InsufficientSecurity".to_string(), + Some(0x50) => "InternalError".to_string(), + Some(0x5A) => "UserCanceled".to_string(), + Some(0x64) => "NoRenegotiation".to_string(), + Some(0x72) => "UnsupportedExtension".to_string(), + Some(opcode) => format!("Unrecognized Alert Description 0x{:02X}", opcode), + None => return format!("Incomplete {}-byte Alert packet", data.len()), + }; + format!("{} {}", level, description) + } + + fn describe_application_data(&self, data: &PlainData) -> String { + if data.len() < 5 { + return "Incomplete ApplicationData".to_string(); + } + format!("{}-byte ApplicationData", data.len() - 5) + } } #[cfg(test)] @@ -796,4 +896,256 @@ mod tests { assert_eq!(None, result); } + + #[test] + fn describe_packet_handles_empty_packet() { + let data = PlainData::new(&[]); + + let result = TlsProtocolPack {}.describe_packet(&data); + + assert_eq!("Incomplete 0-byte TLS packet", result); + } + + #[test] + fn describe_packet_handles_unrecognized_packet_type() { + let data = PlainData::new(&[0xFFu8]); + + let result = TlsProtocolPack {}.describe_packet(&data); + + assert_eq!("1-byte packet of unrecognized type 0xFF", result); + } + + #[test] + fn describe_packet_handles_short_handshake_packet() { + let data = PlainData::new(&[0x16u8]); + + let result = TlsProtocolPack {}.describe_packet(&data); + + assert_eq!("Incomplete 1-byte Handshake packet", result); + } + + #[test] + fn identifies_client_hello_with_sni() { + #[rustfmt::skip] + let data = PlainData::new(&[ + 0x16, // content_type: Handshake + 0x00, 0x00, 0x00, 0x00, // version, length: don't care + 0x01, // handshake_type: ClientHello + 0x00, 0x00, 0x00, 0x00, 0x00, // length, version: don't care + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // random: don't care + 0x01, // session_id_length + 0x00, // session_id: don't care + 0x00, 0x01, // cipher_suites_length + 0x00, // cipher_suite: don't care + 0x01, // compression_methods_length + 0x00, // compression_method: don't care + 0x00, 0x13, // extensions_length + 0x00, 0x00, // extension_type: server_name + 0x00, 0x0F, // extension_length + 0x00, 0x0D, // server_name_list_length + 0x00, // server_name_type + 0x00, 0x0A, // server_name_length + b's', b'e', b'r', b'v', b'e', b'r', b'.', b'c', + b'o', b'm', // server_name + ]); + + let result = TlsProtocolPack {}.describe_packet(&data); + + assert_eq!("ClientHello with SNI 'server.com'", result); + } + + #[test] + fn identifies_client_hello_without_sni() { + #[rustfmt::skip] + let data = PlainData::new(&[ + 0x16, // content_type: Handshake + 0x00, 0x00, 0x00, 0x00, // version, length: don't care + 0x01, // handshake_type: ClientHello + 0x00, 0x00, 0x00, 0x00, 0x00, // length, version: don't care + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // random: don't care + 0x00, // session_id_length + 0x00, 0x00, // cipher_suites_length + 0x00, // compression_methods_length + 0x00, 0x00, // extensions_length + ]); + + let result = TlsProtocolPack {}.describe_packet(&data); + + assert_eq!("ClientHello with no SNI extension", result); + } + + #[test] + fn identifies_other_handshakes() { + #[rustfmt::skip] + let mut bytes: Vec = vec![ + 0x16, // content_type: Handshake + 0x00, 0x00, 0x00, 0x00, // version, length: don't care + 0x00, // handshake_type: replace me + ]; + let handshake_types = vec![ + (0x00, "HelloRequest"), + (0x02, "ServerHello"), + (0x03, "HelloVerifyRequest"), + (0x04, "NewSessionTicket"), + (0x05, "EndOfEarlyData"), + (0x06, "HelloRetryRequest"), + (0x07, "Unrecognized Handshake 0x07"), + (0x08, "Unrecognized Handshake 0x08"), + (0x0B, "Certificate"), + (0x0C, "ServerKeyExchange"), + (0x0D, "CertificateRequest"), + (0x0E, "ServerHelloDone"), + (0x0F, "CertificateVerify"), + (0x10, "ClientKeyExchange"), + (0x14, "Finished"), + ]; + handshake_types.iter().for_each(|(opcode, name)| { + bytes[5] = *opcode; + let data = PlainData::new(&bytes); + + let result = TlsProtocolPack {}.describe_packet(&data); + + assert_eq!(*name, result); + }); + } + + #[test] + fn identifies_cipher_spec_packets() { + #[rustfmt::skip] + let mut bytes: Vec = vec![ + 0x14, // content_type: ChangeCipherSpec + 0x00, 0x00, 0x00, 0x00, // version, length: don't care + 0x00, // change_cipher_spec: replace me + ]; + let change_cipher_specs = vec![ + (0x00, "Unrecognized ChangeCipherSpec 0x00"), + (0x01, "ChangeCipherSpec"), + (0x02, "Unrecognized ChangeCipherSpec 0x02"), + ]; + change_cipher_specs.iter().for_each(|(opcode, name)| { + bytes[5] = *opcode; + let data = PlainData::new(&bytes); + + let result = TlsProtocolPack {}.describe_packet(&data); + + assert_eq!(*name, result); + }); + } + + #[test] + fn handles_incomplete_cipher_spec_packet() { + #[rustfmt::skip] + let data = PlainData::new(&[ + 0x14, // content_type: ChangeCipherSpec + ]); + + let result = TlsProtocolPack {}.describe_packet(&data); + + assert_eq!("Incomplete 1-byte ChangeCipherSpec packet", result); + } + + #[test] + fn identifies_alert_packets() { + #[rustfmt::skip] + let mut bytes: Vec = vec![ + 0x15, // content_type: Alert + 0x00, 0x00, 0x00, 0x00, // version, length: don't care + 0x00, // alert_level: replace me + 0x00, // alert_description: replace me + ]; + let alert_levels = vec![ + (0x01, "Warning"), + (0x02, "Fatal"), + (0x03, "Unrecognized Alert Level 0x03"), + (0x04, "Unrecognized Alert Level 0x04"), + ]; + let alert_descriptions = vec![ + (0x00, "CloseNotify"), + (0x01, "Unrecognized Alert Description 0x01"), + (0x0A, "UnexpectedMessage"), + (0x14, "BadRecordMAC"), + (0x15, "DecryptionFailed"), + (0x16, "RecordOverflow"), + (0x1E, "DecompressionFailure"), + (0x28, "HandshakeFailure"), + (0x29, "NoCertificate"), + (0x2A, "BadCertificate"), + (0x2B, "UnsupportedCertificate"), + (0x2C, "CertificateRevoked"), + (0x2D, "CertificateExpired"), + (0x2E, "CertificateUnknown"), + (0x2F, "IllegalParameter"), + (0x30, "UnknownCA"), + (0x31, "AccessDenied"), + (0x32, "DecodeError"), + (0x33, "DecryptError"), + (0x3C, "ExportRestriction"), + (0x46, "ProtocolVersion"), + (0x47, "InsufficientSecurity"), + (0x50, "InternalError"), + (0x5A, "UserCanceled"), + (0x64, "NoRenegotiation"), + (0x72, "UnsupportedExtension"), + (0xFF, "Unrecognized Alert Description 0xFF"), + ]; + alert_descriptions + .iter() + .for_each(|(description, description_name)| { + bytes[6] = *description; + alert_levels.iter().for_each(|(level, level_name)| { + bytes[5] = *level; + let data = PlainData::new(&bytes); + + let result = TlsProtocolPack {}.describe_packet(&data); + + let expected = format!("{} {}", level_name, description_name); + assert_eq!(expected, result); + }); + }); + } + + #[test] + fn handles_incomplete_alert_packet() { + #[rustfmt::skip] + let data = PlainData::new(&[ + 0x15, // content_type: Alert + ]); + + let result = TlsProtocolPack {}.describe_packet(&data); + + assert_eq!("Incomplete 1-byte Alert packet", result); + } + + #[test] + fn identifies_application_data_packets() { + #[rustfmt::skip] + let data = PlainData::new(&[ + 0x17, // content_type: ApplicationData + 0x00, 0x00, 0x00, 0x00, // version, length: don't care + 0x01, 0x02, 0x03, 0x04, 0x05, // data + ]); + + let result = TlsProtocolPack {}.describe_packet(&data); + + assert_eq!("5-byte ApplicationData", result); + } + + #[test] + fn handles_short_application_data_packets() { + #[rustfmt::skip] + let data = PlainData::new(&[ + 0x17, // content_type: ApplicationData + 0x00, 0x00, 0x00, // Too short by one byte + ]); + + let result = TlsProtocolPack {}.describe_packet(&data); + + assert_eq!("Incomplete ApplicationData", result); + } } diff --git a/node/src/server_initializer.rs b/node/src/server_initializer.rs index 8326e0157..0dcdecda4 100644 --- a/node/src/server_initializer.rs +++ b/node/src/server_initializer.rs @@ -821,7 +821,11 @@ pub mod tests { ]), ); - assert!(result.is_ok()); + assert!( + result.is_ok(), + "Result should have been Ok(()), but was: {:?}", + result + ); let real_user = RealUser::new(Some(123), Some(456), Some("/home/alice".into())); let chown_params = chown_params_arc.lock().unwrap(); assert_eq!( diff --git a/node/src/stream_handler_pool.rs b/node/src/stream_handler_pool.rs index 3e929140c..c715e64df 100644 --- a/node/src/stream_handler_pool.rs +++ b/node/src/stream_handler_pool.rs @@ -516,7 +516,7 @@ impl StreamHandlerPool { sw_key ); debug!(self.logger, "Masking {} bytes", msg.context.data.len()); - let packet = if msg.context.sequence_number.is_none() { + let packet = if msg.context.sequence_number_opt.is_none() { let masquerader = self.traffic_analyzer.get_masquerader(); match masquerader.mask(msg.context.data.as_slice()) { Ok(masked_data) => SequencedPacket::new(masked_data, 0, false), @@ -849,7 +849,7 @@ mod tests { let peer_addr = SocketAddr::from_str("1.2.3.4:80").unwrap(); let peer_addr_a = peer_addr.clone(); let local_addr = SocketAddr::from_str("1.2.3.5:80").unwrap(); - let reception_port = Some(8081); + let reception_port_opt = Some(8081); let is_clandestine = false; let one_http_req = b"GET http://here.com HTTP/1.1\r\n\r\n".to_vec(); let one_http_req_a = one_http_req.clone(); @@ -901,7 +901,7 @@ mod tests { .add_sub .try_send(AddStreamMsg::new( connection_info, // the stream splitter mock will return mocked reader/writer - reception_port, + reception_port_opt, PortConfiguration::new( vec![Box::new(HttpRequestDiscriminatorFactory::new())], is_clandestine, @@ -922,10 +922,10 @@ mod tests { &dispatcher::InboundClientData { timestamp: dispatcher_record.timestamp, client_addr: peer_addr_a, - reception_port, + reception_port_opt, last_data: false, is_clandestine, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: one_http_req_a, } ); @@ -936,10 +936,10 @@ mod tests { &dispatcher::InboundClientData { timestamp: dispatcher_record.timestamp, client_addr: peer_addr_a, - reception_port, + reception_port_opt, last_data: false, is_clandestine, - sequence_number: Some(1), + sequence_number_opt: Some(1), data: another_http_req_a, } ); @@ -950,10 +950,10 @@ mod tests { &dispatcher::InboundClientData { timestamp: dispatcher_record.timestamp, client_addr: peer_addr_a, - reception_port, + reception_port_opt, last_data: false, is_clandestine, - sequence_number: Some(2), + sequence_number_opt: Some(2), data: a_third_http_req_a, } ); @@ -963,7 +963,7 @@ mod tests { &dispatcher::StreamShutdownMsg { peer_addr: peer_addr_a, stream_type: RemovedStreamType::NonClandestine(NonClandestineAttributes { - reception_port: reception_port.unwrap(), + reception_port: reception_port_opt.unwrap(), sequence_number: 3 }), report_to_counterpart: true, @@ -1025,7 +1025,7 @@ mod tests { .try_send(TransmitDataMsg { endpoint: Endpoint::Socket(peer_addr), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: b"hello".to_vec(), }) .unwrap(); @@ -1107,7 +1107,7 @@ mod tests { .try_send(TransmitDataMsg { endpoint: Endpoint::Socket(peer_addr), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: vec![0x12, 0x34], }) .unwrap(); @@ -1130,7 +1130,7 @@ mod tests { .try_send(TransmitDataMsg { endpoint: Endpoint::Socket(peer_addr), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: vec![0x56, 0x78], }) .unwrap(); @@ -1209,7 +1209,7 @@ mod tests { .try_send(TransmitDataMsg { endpoint: Endpoint::Socket(peer_addr), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: vec![0x12, 0x34], }) .unwrap(); @@ -1381,7 +1381,7 @@ mod tests { context: TransmitDataMsg { endpoint: Endpoint::Key(public_key), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: b"hello".to_vec(), }, }) @@ -1470,7 +1470,7 @@ mod tests { .try_send(TransmitDataMsg { endpoint: Endpoint::Key(public_key.clone()), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: outgoing_unmasked, }) .unwrap(); @@ -1505,10 +1505,10 @@ mod tests { &InboundClientData { timestamp: ibcd.timestamp, client_addr: SocketAddr::from_str("1.2.3.5:7000").unwrap(), - reception_port: Some(54321), + reception_port_opt: Some(54321), last_data: false, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: incoming_unmasked, } ); @@ -1586,7 +1586,7 @@ mod tests { .try_send(TransmitDataMsg { endpoint: Endpoint::Key(key.clone()), last_data: false, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: b"hello".to_vec(), }) .unwrap(); @@ -1646,7 +1646,7 @@ mod tests { context: TransmitDataMsg { endpoint: Endpoint::Key(key.clone()), last_data: false, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: b"hello".to_vec(), }, }) @@ -1698,7 +1698,7 @@ mod tests { context: TransmitDataMsg { endpoint: Endpoint::Key(key.clone()), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: b"hello".to_vec(), }, }) @@ -1728,7 +1728,7 @@ mod tests { let msg = TransmitDataMsg { endpoint: Endpoint::Socket(peer_addr.clone()), last_data: false, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: b"hello".to_vec(), }; let msg_a = msg.clone(); @@ -1840,7 +1840,7 @@ mod tests { context: TransmitDataMsg { endpoint: Endpoint::Socket(peer_addr.clone()), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: b"hello".to_vec(), }, }; @@ -1869,13 +1869,13 @@ mod tests { let msg = TransmitDataMsg { endpoint: Endpoint::Socket(peer_addr.clone()), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: b"hello".to_vec(), }; let msg_a = TransmitDataMsg { endpoint: Endpoint::Socket(peer_addr.clone()), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: b"worlds".to_vec(), }; let expected_data = JsonMasquerader::new().mask(&msg_a.data).unwrap(); @@ -1995,7 +1995,7 @@ mod tests { context: TransmitDataMsg { endpoint: Endpoint::Socket(peer_addr.clone()), last_data: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: b"hello".to_vec(), }, }); @@ -2035,7 +2035,7 @@ mod tests { let msg = TransmitDataMsg { endpoint: Endpoint::Socket(peer_addr.clone()), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: b"hello".to_vec(), }; @@ -2126,7 +2126,7 @@ mod tests { .try_send(TransmitDataMsg { endpoint: Endpoint::Socket(peer_addr), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: hello, }) .unwrap(); @@ -2136,7 +2136,7 @@ mod tests { .try_send(TransmitDataMsg { endpoint: Endpoint::Socket(peer_addr), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: worlds, }) .unwrap(); @@ -2200,7 +2200,7 @@ mod tests { .try_send(TransmitDataMsg { endpoint: Endpoint::Socket(peer_addr), last_data: false, - sequence_number: None, + sequence_number_opt: None, data: b"hello".to_vec(), }) .unwrap(); @@ -2249,7 +2249,7 @@ mod tests { .try_send(TransmitDataMsg { endpoint: Endpoint::Socket(local_addr), last_data: false, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: outgoing_unmasked, }) .unwrap(); diff --git a/node/src/stream_reader.rs b/node/src/stream_reader.rs index cac9211a6..df8dd8f33 100644 --- a/node/src/stream_reader.rs +++ b/node/src/stream_reader.rs @@ -19,7 +19,7 @@ pub struct StreamReaderReal { stream: Box, local_addr: SocketAddr, peer_addr: SocketAddr, - reception_port: Option, + reception_port_opt: Option, ibcd_sub: Recipient, remove_sub: Recipient, dispatcher_stream_shutdown_sub: Recipient, @@ -86,7 +86,7 @@ impl StreamReaderReal { #[allow(clippy::too_many_arguments)] pub fn new( stream: Box, - reception_port: Option, + reception_port_opt: Option, ibcd_sub: Recipient, remove_sub: Recipient, dispatcher_sub: Recipient, @@ -107,7 +107,7 @@ impl StreamReaderReal { stream, local_addr, peer_addr, - reception_port, + reception_port_opt, ibcd_sub, remove_sub, dispatcher_stream_shutdown_sub: dispatcher_sub, @@ -137,7 +137,7 @@ impl StreamReaderReal { // handshake and should start the sequence at Some(0) as well, the ProxyServer will // handle the sequenced packet offset before sending them through the stream_writer // and avoid dropping duplicate packets. - let sequence_number = if unmasked_chunk.sequenced && !is_connect { + let sequence_number_opt = if unmasked_chunk.sequenced && !is_connect { Some(self.sequencer.next_sequence_number()) } else if is_connect { // This case needs to explicitly be Some(0) instead of None so that the StreamHandlerPool does @@ -146,7 +146,7 @@ impl StreamReaderReal { } else { None }; - match sequence_number { + match sequence_number_opt { Some(num) => debug!( self.logger, "Read {} bytes of clear data (#{})", @@ -162,10 +162,10 @@ impl StreamReaderReal { let msg = dispatcher::InboundClientData { timestamp: SystemTime::now(), client_addr: self.peer_addr, - reception_port: self.reception_port, + reception_port_opt: self.reception_port_opt, last_data: false, is_clandestine: self.is_clandestine, - sequence_number, + sequence_number_opt, data: unmasked_chunk.chunk.clone(), }; debug!(self.logger, "Discriminator framed and unmasked {} bytes for {}; transmitting via Hopper", @@ -181,7 +181,7 @@ impl StreamReaderReal { } fn shutdown(&mut self) { - debug!(self.logger, "Directing removal of {}clandestine StreamReader with reception_port {:?} on {} listening to {}", if self.is_clandestine {""} else {"non-"}, self.reception_port, self.local_addr, self.peer_addr); + debug!(self.logger, "Directing removal of {}clandestine StreamReader with reception_port {:?} on {} listening to {}", if self.is_clandestine {""} else {"non-"}, self.reception_port_opt, self.local_addr, self.peer_addr); self.remove_sub .try_send(RemoveStreamMsg { peer_addr: self.peer_addr, @@ -190,7 +190,7 @@ impl StreamReaderReal { RemovedStreamType::Clandestine } else { RemovedStreamType::NonClandestine(NonClandestineAttributes { - reception_port: self.reception_port.expect( + reception_port: self.reception_port_opt.expect( "Non-clandestine StreamReader should always have a reception_port", ), sequence_number: self.sequencer.next_sequence_number(), @@ -511,10 +511,10 @@ mod tests { &dispatcher::InboundClientData { timestamp: d_record.timestamp, client_addr: peer_addr, - reception_port: Some(1234 as u16), + reception_port_opt: Some(1234 as u16), last_data: false, is_clandestine: true, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: Vec::from("GET http://here.com HTTP/1.1\r\n\r\n".as_bytes()), } ); @@ -575,13 +575,13 @@ mod tests { Some(0), d_recording .get_record::(0) - .sequence_number, + .sequence_number_opt, ); assert_eq!( Some(0), d_recording .get_record::(1) - .sequence_number, + .sequence_number_opt, ); } @@ -633,10 +633,10 @@ mod tests { &dispatcher::InboundClientData { timestamp: d_record.timestamp, client_addr: peer_addr, - reception_port: Some(1234 as u16), + reception_port_opt: Some(1234 as u16), last_data: false, is_clandestine: false, - sequence_number: Some(0), + sequence_number_opt: Some(0), data: Vec::from("GET http://here.com HTTP/1.1\r\n\r\n".as_bytes()), } ); @@ -648,10 +648,10 @@ mod tests { &dispatcher::InboundClientData { timestamp: d_record.timestamp, client_addr: peer_addr, - reception_port: Some(1234 as u16), + reception_port_opt: Some(1234 as u16), last_data: false, is_clandestine: false, - sequence_number: Some(1), + sequence_number_opt: Some(1), data: Vec::from("GET http://www.example.com HTTP/1.1\r\n\r\n".as_bytes()), } ); @@ -707,10 +707,10 @@ mod tests { &dispatcher::InboundClientData { timestamp: d_record.timestamp, client_addr, - reception_port: Some(1234 as u16), + reception_port_opt: Some(1234 as u16), last_data: false, is_clandestine: true, - sequence_number: None, + sequence_number_opt: None, data: Vec::from("GET http://here.com HTTP/1.1\r\n\r\n".as_bytes()), } ); diff --git a/node/src/sub_lib/combined_parameters.rs b/node/src/sub_lib/combined_parameters.rs index bd26eb627..6b181a8d5 100644 --- a/node/src/sub_lib/combined_parameters.rs +++ b/node/src/sub_lib/combined_parameters.rs @@ -13,7 +13,7 @@ use std::fmt; use std::fmt::Display; use std::time::Duration; -macro_rules! initiate_struct{ +macro_rules! initialize_struct{ ($struct_type: ident, $hash_map: expr, $($field:literal),+) =>{ paste!{ $struct_type{ @@ -119,32 +119,29 @@ impl CombinedParams { delimiter: char, expected_collection: &[(&str, CombinedParamsDataTypes)], ) -> Result, String> { - let check = |count: usize| { - if count != expected_collection.len() { - return Err(format!( - "Wrong number of values: expected {} but {} supplied{}", - expected_collection.len(), - count, - if count == 1 { - format!(". Did you use the correct delimiter '{}'?", delimiter) - } else { - "".to_string() - } - )); - } - Ok(()) - }; let pieces: Vec<&str> = input.split(delimiter).collect(); - check(pieces.len())?; + let param_count = pieces.len(); + let expected_param_count = expected_collection.len(); + if param_count != expected_param_count { + return Err(format!( + "Wrong number of values: expected {} but {} supplied{}", + expected_param_count, + param_count, + if param_count == 1 { + format!(". Did you use the correct delimiter '{}'?", delimiter) + } else { + "".to_string() + } + )); + } let zipped = pieces.into_iter().zip(expected_collection.iter()); - Ok(zipped - .map(|(piece, (param_name, data_type))| { - ( - param_name.to_string(), - CombinedParamsValueRetriever::parse(piece, data_type).expectv("numeric value"), - ) - }) - .collect()) + let remapped_pairs = zipped.map(|(piece, (param_name, data_type))| { + ( + param_name.to_string(), + CombinedParamsValueRetriever::parse(piece, data_type).expectv("numeric value"), + ) + }); + Ok(HashMap::from_iter(remapped_pairs)) } fn initialize_objects( @@ -152,7 +149,7 @@ impl CombinedParams { parsed_values: HashMap, ) -> Self { match self { - Self::RatePack(Uninitialized) => Self::RatePack(Initialized(initiate_struct!( + Self::RatePack(Uninitialized) => Self::RatePack(Initialized(initialize_struct!( RatePack, &parsed_values, "routing_byte_rate", @@ -161,7 +158,7 @@ impl CombinedParams { "exit_service_rate" ))), Self::PaymentThresholds(Uninitialized) => { - Self::PaymentThresholds(Initialized(initiate_struct!( + Self::PaymentThresholds(Initialized(initialize_struct!( PaymentThresholds, &parsed_values, "maturity_threshold_sec", @@ -173,7 +170,7 @@ impl CombinedParams { ))) } Self::ScanIntervals(Uninitialized) => { - Self::ScanIntervals(Initialized(initiate_struct!( + Self::ScanIntervals(Initialized(initialize_struct!( ScanIntervals, &parsed_values, Duration::from_secs, @@ -183,7 +180,7 @@ impl CombinedParams { ))) } _ => panic!( - "should be called only on uninitialized object, not: {:?}", + "should be called only on an uninitialized object, not: {:?}", self ), } @@ -213,7 +210,7 @@ impl From<&CombinedParams> for &[(&str, CombinedParamsDataTypes)] { ("receivable_scan_interval", U64), ], _ => panic!( - "should be called only on uninitialized object, not: {:?}", + "should be called only on an uninitialized object, not: {:?}", params ), } @@ -433,7 +430,7 @@ mod tests { assert_eq!( panic_1_msg, &format!( - "should be called only on uninitialized object, not: RatePack(Initialized({:?}))", + "should be called only on an uninitialized object, not: RatePack(Initialized({:?}))", DEFAULT_RATE_PACK ) ); @@ -449,7 +446,7 @@ mod tests { assert_eq!( panic_2_msg, &format!( - "should be called only on uninitialized object, not: PaymentThresholds(Initialized({:?}))", + "should be called only on an uninitialized object, not: PaymentThresholds(Initialized({:?}))", PaymentThresholds::default() ) ); @@ -464,7 +461,7 @@ mod tests { assert_eq!( panic_3_msg, &format!( - "should be called only on uninitialized object, not: ScanIntervals(Initialized({:?}))", + "should be called only on an uninitialized object, not: ScanIntervals(Initialized({:?}))", *TEST_SCAN_INTERVALS ) ); @@ -482,7 +479,7 @@ mod tests { assert_eq!( panic_1_msg, &format!( - "should be called only on uninitialized object, not: RatePack(Initialized({:?}))", + "should be called only on an uninitialized object, not: RatePack(Initialized({:?}))", DEFAULT_RATE_PACK ) ); @@ -497,7 +494,7 @@ mod tests { assert_eq!( panic_2_msg, &format!( - "should be called only on uninitialized object, not: PaymentThresholds(Initialized({:?}))", + "should be called only on an uninitialized object, not: PaymentThresholds(Initialized({:?}))", PaymentThresholds::default() ) ); @@ -512,7 +509,7 @@ mod tests { assert_eq!( panic_3_msg, &format!( - "should be called only on uninitialized object, not: ScanIntervals(Initialized({:?}))", + "should be called only on an uninitialized object, not: ScanIntervals(Initialized({:?}))", *TEST_SCAN_INTERVALS ) ); diff --git a/node/src/sub_lib/dispatcher.rs b/node/src/sub_lib/dispatcher.rs index 66e349e11..e04401220 100644 --- a/node/src/sub_lib/dispatcher.rs +++ b/node/src/sub_lib/dispatcher.rs @@ -116,10 +116,10 @@ pub enum DispatcherError { pub struct InboundClientData { pub timestamp: SystemTime, pub client_addr: SocketAddr, - pub reception_port: Option, + pub reception_port_opt: Option, pub last_data: bool, pub is_clandestine: bool, - pub sequence_number: Option, + pub sequence_number_opt: Option, pub data: Vec, } @@ -130,7 +130,7 @@ impl Debug for InboundClientData { Err(_) => self.data.hex_dump().to_string(), }; write!(f, "InboundClientData {{ peer_addr: {:?}, reception_port: {:?}, last_data: {}, sequence_number: {:?}, {} bytes of data: {} }}", - self.client_addr, self.reception_port, self.last_data, self.sequence_number, self.data.len(), data_string) + self.client_addr, self.reception_port_opt, self.last_data, self.sequence_number_opt, self.data.len(), data_string) } } @@ -139,10 +139,10 @@ impl InboundClientData { InboundClientData { timestamp: SystemTime::now(), client_addr: self.client_addr, - reception_port: self.reception_port, + reception_port_opt: self.reception_port_opt, last_data: self.last_data, is_clandestine: self.is_clandestine, - sequence_number: self.sequence_number, + sequence_number_opt: self.sequence_number_opt, data: vec![], } } @@ -274,10 +274,10 @@ mod tests { let subject = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.4.3.2:9999").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: false, is_clandestine: false, - sequence_number: None, + sequence_number_opt: None, data: b"CONNECT server.example.com:80 HTTP/1.1\r\nHost: server.example.com:80\r\nProxy-Authorization: basic aGVsbG86d29ybGQ=\r\n\r\n".to_vec(), }; @@ -289,10 +289,10 @@ mod tests { let subject = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.4.3.2:9999").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: false, is_clandestine: false, - sequence_number: None, + sequence_number_opt: None, data: b"GET server.example.com:80 HTTP/1.1\r\nHost: server.example.com:80\r\nProxy-Authorization: basic aGVsbG86d29ybGQ=\r\n\r\n".to_vec(), }; @@ -304,10 +304,10 @@ mod tests { let subject = InboundClientData { timestamp: SystemTime::now(), client_addr: SocketAddr::from_str("1.4.3.2:9999").unwrap(), - reception_port: None, + reception_port_opt: None, last_data: false, is_clandestine: false, - sequence_number: None, + sequence_number_opt: None, data: b"CONNECTX".to_vec(), }; diff --git a/node/src/sub_lib/hopper.rs b/node/src/sub_lib/hopper.rs index 96d756ef2..08af13bb5 100644 --- a/node/src/sub_lib/hopper.rs +++ b/node/src/sub_lib/hopper.rs @@ -173,6 +173,7 @@ mod tests { use crate::sub_lib::cryptde::PlainData; use crate::sub_lib::dispatcher::Component; use crate::sub_lib::route::RouteSegment; + use crate::sub_lib::stream_key::StreamKey; use crate::test_utils::recorder::Recorder; use crate::test_utils::{make_meaningless_message_type, make_paying_wallet}; use actix::Actor; @@ -205,7 +206,7 @@ mod tests { let cryptde = CRYPTDE_PAIR.main.as_ref(); let public_key = PublicKey::new(&[1, 2]); let node_addr = NodeAddr::new(&IpAddr::from_str("1.2.3.4").unwrap(), &[1, 2, 3, 4]); - let payload = make_meaningless_message_type(); + let payload = make_meaningless_message_type(StreamKey::make_meaningless_stream_key()); let result = NoLookupIncipientCoresPackage::new(cryptde, &public_key, &node_addr, payload.clone()); @@ -231,7 +232,7 @@ mod tests { cryptde, &PublicKey::new(&[]), &NodeAddr::new(&IpAddr::from_str("1.1.1.1").unwrap(), &[]), - make_meaningless_message_type(), + make_meaningless_message_type(StreamKey::make_meaningless_stream_key()), ); assert_eq!( result, @@ -255,7 +256,7 @@ mod tests { Some(TEST_DEFAULT_CHAIN.rec().contract), ) .unwrap(); - let payload = make_meaningless_message_type(); + let payload = make_meaningless_message_type(StreamKey::make_meaningless_stream_key()); let result = IncipientCoresPackage::new(cryptde, route.clone(), payload.clone(), &key56); let subject = result.unwrap(); @@ -278,7 +279,7 @@ mod tests { let result = IncipientCoresPackage::new( cryptde, Route { hops: vec![] }, - make_meaningless_message_type(), + make_meaningless_message_type(StreamKey::make_meaningless_stream_key()), &PublicKey::new(&[]), ); @@ -304,7 +305,7 @@ mod tests { Some(TEST_DEFAULT_CHAIN.rec().contract), ) .unwrap(); - let payload = make_meaningless_message_type(); + let payload = make_meaningless_message_type(StreamKey::make_meaningless_stream_key()); let subject: ExpiredCoresPackage = ExpiredCoresPackage::new( immediate_neighbor, diff --git a/node/src/sub_lib/host.rs b/node/src/sub_lib/host.rs new file mode 100644 index 000000000..bb23d8d16 --- /dev/null +++ b/node/src/sub_lib/host.rs @@ -0,0 +1,39 @@ +use std::fmt::Display; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Host { + pub name: String, + pub port: u16, +} + +impl Display for Host { + fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { + write!(f, "{}:{}", self.name, self.port) + } +} + +impl Host { + pub fn new(name: &str, port: u16) -> Host { + Host { + name: name.to_string(), + port, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display() { + let subject = Host { + name: "example.com".to_string(), + port: 8080, + }; + + let result = format!("{}", subject); + + assert_eq!(result, "example.com:8080".to_string()); + } +} diff --git a/node/src/sub_lib/http_packet_framer.rs b/node/src/sub_lib/http_packet_framer.rs index 29685f46b..28d4f895f 100644 --- a/node/src/sub_lib/http_packet_framer.rs +++ b/node/src/sub_lib/http_packet_framer.rs @@ -104,7 +104,7 @@ impl HttpPacketFramer { lines: Vec::new(), }, start_finder, - logger: Logger::new("HttpRequestFramer"), + logger: Logger::new("HttpPacketFramer"), } } diff --git a/node/src/sub_lib/migrations/client_request_payload.rs b/node/src/sub_lib/migrations/client_request_payload.rs index 2bd9993f5..b048c1900 100644 --- a/node/src/sub_lib/migrations/client_request_payload.rs +++ b/node/src/sub_lib/migrations/client_request_payload.rs @@ -49,7 +49,7 @@ impl TryFrom<&Value> for ClientRequestPayload_0v1 { Value::Map(map) => { let mut stream_key_opt: Option = None; let mut sequenced_packet_opt: Option = None; - let mut target_hostname_opt: Option> = None; + let mut target_hostname: Option = None; let mut target_port_opt: Option = None; let mut protocol_opt: Option = None; let mut originator_public_key_opt: Option = None; @@ -61,9 +61,7 @@ impl TryFrom<&Value> for ClientRequestPayload_0v1 { "sequenced_packet" => { sequenced_packet_opt = value_to_type::(v) } - "target_hostname" => { - target_hostname_opt = value_to_type::>(v) - } + "target_hostname" => target_hostname = value_to_type::(v), "target_port" => target_port_opt = value_to_type::(v), "protocol" => protocol_opt = value_to_type::(v), "originator_public_key" => { @@ -89,7 +87,7 @@ impl TryFrom<&Value> for ClientRequestPayload_0v1 { "sequenced_packet", &sequenced_packet_opt, ); - check_field(&mut missing_fields, "target_hostname", &target_hostname_opt); + check_field(&mut missing_fields, "target_hostname", &target_hostname); check_field(&mut missing_fields, "target_port", &target_port_opt); check_field(&mut missing_fields, "protocol", &protocol_opt); check_field( @@ -103,7 +101,7 @@ impl TryFrom<&Value> for ClientRequestPayload_0v1 { Ok(ClientRequestPayload_0v1 { stream_key: stream_key_opt.expect("stream_key disappeared"), sequenced_packet: sequenced_packet_opt.expect("sequenced_packet disappeared"), - target_hostname: target_hostname_opt.expect("target_hostname disappeared"), + target_hostname: target_hostname.expect("target_hostname disappeared"), target_port: target_port_opt.expect("target_port disappeared"), protocol: protocol_opt.expect("protocol disappeared"), originator_public_key: originator_public_key_opt @@ -131,7 +129,7 @@ mod tests { struct ExampleFutureCRP { pub stream_key: StreamKey, pub sequenced_packet: SequencedPacket, - pub target_hostname: Option, + pub target_hostname: String, pub target_port: u16, pub protocol: ProxyProtocol, pub originator_public_key: PublicKey, @@ -141,7 +139,7 @@ mod tests { let expected_crp = ClientRequestPayload_0v1 { stream_key: StreamKey::make_meaningful_stream_key("All Things Must Pass"), sequenced_packet: SequencedPacket::new(vec![4, 3, 2, 1], 4321, false), - target_hostname: Some("target.hostname.com".to_string()), + target_hostname: "target.hostname.com".to_string(), target_port: 1234, protocol: ProxyProtocol::HTTP, originator_public_key: PublicKey::new(&[2, 3, 4, 5]), diff --git a/node/src/sub_lib/mod.rs b/node/src/sub_lib/mod.rs index 51360357b..4c1a76fce 100644 --- a/node/src/sub_lib/mod.rs +++ b/node/src/sub_lib/mod.rs @@ -21,6 +21,7 @@ pub mod framer; pub mod framer_utils; pub mod hop; pub mod hopper; +pub mod host; pub mod http_packet_framer; pub mod http_response_start_finder; pub mod limiter; diff --git a/node/src/sub_lib/neighborhood.rs b/node/src/sub_lib/neighborhood.rs index f26282aa6..798020cf0 100644 --- a/node/src/sub_lib/neighborhood.rs +++ b/node/src/sub_lib/neighborhood.rs @@ -8,6 +8,7 @@ use crate::sub_lib::cryptde::{CryptDE, PublicKey}; use crate::sub_lib::cryptde_real::CryptDEReal; use crate::sub_lib::dispatcher::{Component, StreamShutdownMsg}; use crate::sub_lib::hopper::ExpiredCoresPackage; +use crate::sub_lib::host::Host; use crate::sub_lib::node_addr::NodeAddr; use crate::sub_lib::peer_actors::{BindMessage, NewPublicIp, StartMessage}; use crate::sub_lib::route::Route; @@ -23,6 +24,7 @@ use lazy_static::lazy_static; use masq_lib::blockchains::blockchain_records::CHAINS; use masq_lib::blockchains::chains::{chain_from_chain_identifier_opt, Chain}; use masq_lib::constants::{CENTRAL_DELIMITER, CHAIN_IDENTIFIER_DELIMITER, MASQ_URL_PREFIX}; +use masq_lib::shared_schema::ConfiguratorError; use masq_lib::ui_gateway::NodeFromUiMessage; use masq_lib::utils::NeighborhoodModeLight; use serde_derive::{Deserialize, Serialize}; @@ -48,6 +50,21 @@ pub const ZERO_RATE_PACK: RatePack = RatePack { exit_service_rate: 0, }; +pub const DEFAULT_RATE_PACK_LIMITS: RatePackLimits = RatePackLimits { + lo: RatePack { + routing_byte_rate: 100, + routing_service_rate: 100, + exit_byte_rate: 100, + exit_service_rate: 100, + }, + hi: RatePack { + routing_byte_rate: 100_000_000_000_000, + routing_service_rate: 100_000_000_000_000, + exit_byte_rate: 100_000_000_000_000, + exit_service_rate: 100_000_000_000_000, + }, +}; + #[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] pub struct RatePack { pub routing_byte_rate: u64, @@ -57,6 +74,20 @@ pub struct RatePack { } impl RatePack { + pub fn new( + routing_byte_rate: u64, + routing_service_rate: u64, + exit_byte_rate: u64, + exit_service_rate: u64, + ) -> Self { + Self { + routing_byte_rate, + routing_service_rate, + exit_byte_rate, + exit_service_rate, + } + } + pub fn routing_charge(&self, payload_size: u64) -> u64 { self.routing_service_rate + (self.routing_byte_rate * payload_size) } @@ -64,6 +95,109 @@ impl RatePack { pub fn exit_charge(&self, payload_size: u64) -> u64 { self.exit_service_rate + (self.exit_byte_rate * payload_size) } + + pub fn rate_pack_parameter(&self) -> String { + format!( + "{}|{}|{}|{}", + self.routing_byte_rate, + self.routing_service_rate, + self.exit_byte_rate, + self.exit_service_rate, + ) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RatePackLimits { + pub lo: RatePack, + pub hi: RatePack, +} + +impl RatePackLimits { + pub fn new(lo: RatePack, hi: RatePack) -> Self { + Self { lo, hi } + } + + pub fn check(&self, rate_pack: &RatePack) -> bool { + self.analyze(rate_pack).is_ok() + } + + pub fn analyze(&self, rate_pack: &RatePack) -> Result<(), ConfiguratorError> { + let check_min_and_max = |candidate: u64, + min: u64, + max: u64, + name: &str, + error: ConfiguratorError| + -> ConfiguratorError { + let mut result = error; + if candidate < min { + result = result.another_required( + "rate-pack", + &format!( + "Value of {} ({}) is below the minimum allowed ({})", + name, candidate, min + ), + ); + } else if candidate > max { + result = result.another_required( + "rate-pack", + &format!( + "Value of {} ({}) is above the maximum allowed ({})", + name, candidate, max + ), + ); + } + result + }; + let mut error = ConfiguratorError::new(vec![]); + error = check_min_and_max( + rate_pack.routing_byte_rate, + self.lo.routing_byte_rate, + self.hi.routing_byte_rate, + "routing_byte_rate", + error, + ); + error = check_min_and_max( + rate_pack.routing_service_rate, + self.lo.routing_service_rate, + self.hi.routing_service_rate, + "routing_service_rate", + error, + ); + error = check_min_and_max( + rate_pack.exit_byte_rate, + self.lo.exit_byte_rate, + self.hi.exit_byte_rate, + "exit_byte_rate", + error, + ); + error = check_min_and_max( + rate_pack.exit_service_rate, + self.lo.exit_service_rate, + self.hi.exit_service_rate, + "exit_service_rate", + error, + ); + if error.is_empty() { + Ok(()) + } else { + Err(error) + } + } + + pub fn rate_pack_limits_parameter(&self) -> String { + format!( + "{}-{}|{}-{}|{}-{}|{}-{}", + self.lo.routing_byte_rate, + self.hi.routing_byte_rate, + self.lo.routing_service_rate, + self.hi.routing_service_rate, + self.lo.exit_byte_rate, + self.hi.exit_byte_rate, + self.lo.exit_service_rate, + self.hi.exit_service_rate, + ) + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -473,7 +607,7 @@ pub struct RouteQueryMessage { pub target_component: Component, pub return_component_opt: Option, pub payload_size: usize, - pub hostname_opt: Option, + pub host: Host, } impl Message for RouteQueryMessage { @@ -481,16 +615,13 @@ impl Message for RouteQueryMessage { } impl RouteQueryMessage { - pub fn data_indefinite_route_request( - hostname_opt: Option, - payload_size: usize, - ) -> RouteQueryMessage { + pub fn data_indefinite_route_request(host: Host, payload_size: usize) -> RouteQueryMessage { RouteQueryMessage { target_key_opt: None, target_component: Component::ProxyClient, return_component_opt: Some(Component::ProxyServer), payload_size, - hostname_opt, + host, } } } @@ -502,16 +633,53 @@ pub enum ExpectedService { Nothing, } +impl ExpectedService { + pub fn exit_node_key_opt(&self) -> Option { + match self { + ExpectedService::Exit(key, _, _) => Some(key.clone()), + _ => None, + } + } + + pub fn public_key_opt(&self) -> Option { + match self { + ExpectedService::Exit(key, _, _) | ExpectedService::Routing(key, _, _) => { + Some(key.clone()) + } + _ => None, + } + } + + pub fn wallet_opt(&self) -> Option<&Wallet> { + match self { + ExpectedService::Exit(_, wallet, _) | ExpectedService::Routing(_, wallet, _) => { + Some(wallet) + } + _ => None, + } + } + + pub fn rate_pack_opt(&self) -> Option<&RatePack> { + match self { + ExpectedService::Exit(_, _, rate_pack) | ExpectedService::Routing(_, _, rate_pack) => { + Some(rate_pack) + } + _ => None, + } + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub enum ExpectedServices { OneWay(Vec), - RoundTrip(Vec, Vec, u32), + RoundTrip(Vec, Vec), } #[derive(Clone, Debug, PartialEq, Eq)] pub struct RouteQueryResponse { pub route: Route, pub expected_services: ExpectedServices, + pub host: Host, } #[derive(Clone, Debug, Message, PartialEq, Eq)] @@ -579,8 +747,8 @@ pub enum GossipFailure_0v1 { Unknown, } -impl fmt::Display for GossipFailure_0v1 { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { +impl Display for GossipFailure_0v1 { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> { let msg = match self { GossipFailure_0v1::NoNeighbors => "No neighbors for Introduction or Pass", GossipFailure_0v1::NoSuitableNeighbors => { @@ -630,7 +798,7 @@ mod tests { use std::str::FromStr; lazy_static! { - static ref CRYPTDE_PAIR: CryptDEPair = CryptDEPair::null(); + static ref NB_CRYPTDE_PAIR: CryptDEPair = CryptDEPair::null(); } #[test] @@ -862,7 +1030,7 @@ mod tests { #[test] fn from_str_complains_about_bad_base_64() { let result = NodeDescriptor::try_from(( - CRYPTDE_PAIR.main.as_ref(), + NB_CRYPTDE_PAIR.main.as_ref(), "masq://eth-mainnet:bad_key@1.2.3.4:1234;2345", )); @@ -904,8 +1072,10 @@ mod tests { #[test] fn from_str_complains_about_blank_public_key() { - let result = - NodeDescriptor::try_from((CRYPTDE_PAIR.main.as_ref(), "masq://dev:@1.2.3.4:1234/2345")); + let result = NodeDescriptor::try_from(( + NB_CRYPTDE_PAIR.main.as_ref(), + "masq://dev:@1.2.3.4:1234/2345", + )); assert_eq!(result, Err(String::from("Public key cannot be empty"))); } @@ -913,7 +1083,7 @@ mod tests { #[test] fn from_str_complains_about_bad_node_addr() { let result = NodeDescriptor::try_from(( - CRYPTDE_PAIR.main.as_ref(), + NB_CRYPTDE_PAIR.main.as_ref(), "masq://eth-mainnet:R29vZEtleQ==@BadNodeAddr", )); @@ -923,7 +1093,7 @@ mod tests { #[test] fn from_str_handles_the_happy_path_with_node_addr() { let result = NodeDescriptor::try_from(( - CRYPTDE_PAIR.main.as_ref(), + NB_CRYPTDE_PAIR.main.as_ref(), "masq://eth-ropsten:R29vZEtleQ@1.2.3.4:1234/2345/3456", )); @@ -943,7 +1113,7 @@ mod tests { #[test] fn from_str_handles_the_happy_path_without_node_addr() { let result = NodeDescriptor::try_from(( - CRYPTDE_PAIR.main.as_ref(), + NB_CRYPTDE_PAIR.main.as_ref(), "masq://eth-mainnet:R29vZEtleQ@:", )); @@ -987,7 +1157,7 @@ mod tests { #[test] fn node_descriptor_from_key_node_addr_and_mainnet_flag_works() { - let cryptde: &dyn CryptDE = CRYPTDE_PAIR.main.as_ref(); + let cryptde: &dyn CryptDE = NB_CRYPTDE_PAIR.main.as_ref(); let public_key = PublicKey::new(&[1, 2, 3, 4, 5, 6, 7, 8]); let node_addr = NodeAddr::new(&IpAddr::from_str("123.45.67.89").unwrap(), &[2345, 3456]); @@ -1005,7 +1175,7 @@ mod tests { #[test] fn node_descriptor_to_string_works_for_mainnet() { - let cryptde: &dyn CryptDE = CRYPTDE_PAIR.main.as_ref(); + let cryptde: &dyn CryptDE = NB_CRYPTDE_PAIR.main.as_ref(); let public_key = PublicKey::new(&[1, 2, 3, 4, 5, 6, 7, 8]); let node_addr = NodeAddr::new(&IpAddr::from_str("123.45.67.89").unwrap(), &[2345, 3456]); let subject = NodeDescriptor::from((&public_key, &node_addr, Chain::EthMainnet, cryptde)); @@ -1020,7 +1190,7 @@ mod tests { #[test] fn node_descriptor_to_string_works_for_not_mainnet() { - let cryptde: &dyn CryptDE = CRYPTDE_PAIR.main.as_ref(); + let cryptde: &dyn CryptDE = NB_CRYPTDE_PAIR.main.as_ref(); let public_key = PublicKey::new(&[1, 2, 3, 4, 5, 6, 7, 8]); let node_addr = NodeAddr::new(&IpAddr::from_str("123.45.67.89").unwrap(), &[2345, 3456]); let subject = NodeDescriptor::from((&public_key, &node_addr, Chain::EthRopsten, cryptde)); @@ -1035,7 +1205,7 @@ mod tests { #[test] fn first_part_of_node_descriptor_must_not_be_longer_than_required() { - let cryptde: &dyn CryptDE = CRYPTDE_PAIR.main.as_ref(); + let cryptde: &dyn CryptDE = NB_CRYPTDE_PAIR.main.as_ref(); let public_key = PublicKey::new(&[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, @@ -1060,7 +1230,8 @@ mod tests { #[test] fn data_indefinite_route_request() { - let result = RouteQueryMessage::data_indefinite_route_request(None, 7500); + let result = + RouteQueryMessage::data_indefinite_route_request(Host::new("booga.com", 1234), 7500); assert_eq!( result, @@ -1069,7 +1240,7 @@ mod tests { target_component: Component::ProxyClient, return_component_opt: Some(Component::ProxyServer), payload_size: 7500, - hostname_opt: None, + host: Host::new("booga.com", 1234), } ); } @@ -1077,12 +1248,12 @@ mod tests { #[test] fn standard_mode_results() { let one_neighbor = NodeDescriptor::try_from(( - CRYPTDE_PAIR.main.as_ref(), + NB_CRYPTDE_PAIR.main.as_ref(), "masq://eth-mainnet:AQIDBA@1.2.3.4:1234", )) .unwrap(); let another_neighbor = NodeDescriptor::try_from(( - CRYPTDE_PAIR.main.as_ref(), + NB_CRYPTDE_PAIR.main.as_ref(), "masq://eth-mainnet:AgMEBQ@2.3.4.5:2345", )) .unwrap(); @@ -1112,12 +1283,12 @@ mod tests { #[test] fn originate_only_mode_results() { let one_neighbor = NodeDescriptor::try_from(( - CRYPTDE_PAIR.main.as_ref(), + NB_CRYPTDE_PAIR.main.as_ref(), "masq://eth-ropsten:AQIDBA@1.2.3.4:1234", )) .unwrap(); let another_neighbor = NodeDescriptor::try_from(( - CRYPTDE_PAIR.main.as_ref(), + NB_CRYPTDE_PAIR.main.as_ref(), "masq://eth-ropsten:AgMEBQ@2.3.4.5:2345", )) .unwrap(); @@ -1143,12 +1314,12 @@ mod tests { #[test] fn consume_only_mode_results() { let one_neighbor = NodeDescriptor::try_from(( - CRYPTDE_PAIR.main.as_ref(), + NB_CRYPTDE_PAIR.main.as_ref(), "masq://eth-mainnet:AQIDBA@1.2.3.4:1234", )) .unwrap(); let another_neighbor = NodeDescriptor::try_from(( - CRYPTDE_PAIR.main.as_ref(), + NB_CRYPTDE_PAIR.main.as_ref(), "masq://eth-mainnet:AgMEBQ@2.3.4.5:2345", )) .unwrap(); diff --git a/node/src/sub_lib/proxy_server.rs b/node/src/sub_lib/proxy_server.rs index c3042859f..c7b0ab49c 100644 --- a/node/src/sub_lib/proxy_server.rs +++ b/node/src/sub_lib/proxy_server.rs @@ -4,7 +4,7 @@ use crate::sub_lib::data_version::DataVersion; use crate::sub_lib::dispatcher::InboundClientData; use crate::sub_lib::dispatcher::StreamShutdownMsg; use crate::sub_lib::hopper::{ExpiredCoresPackage, MessageType}; -use crate::sub_lib::neighborhood::{ExpectedService, RouteQueryResponse}; +use crate::sub_lib::neighborhood::RouteQueryResponse; use crate::sub_lib::peer_actors::BindMessage; use crate::sub_lib::proxy_client::{ClientResponsePayload_0v1, DnsResolveFailure_0v1}; use crate::sub_lib::sequence_buffer::SequencedPacket; @@ -34,7 +34,7 @@ pub enum ProxyProtocol { pub struct ClientRequestPayload_0v1 { pub stream_key: StreamKey, pub sequenced_packet: SequencedPacket, - pub target_hostname: Option, + pub target_hostname: String, pub target_port: u16, pub protocol: ProxyProtocol, pub originator_public_key: PublicKey, @@ -55,14 +55,6 @@ impl ClientRequestPayload_0v1 { } } -#[derive(Message, Debug, PartialEq, Eq)] -pub struct AddReturnRouteMessage { - pub return_route_id: u32, - pub expected_services: Vec, - pub protocol: ProxyProtocol, - pub hostname_opt: Option, -} - #[derive(Message, Debug, PartialEq, Eq)] pub struct AddRouteResultMessage { pub stream_key: StreamKey, @@ -81,7 +73,6 @@ pub struct ProxyServerSubs { pub from_dispatcher: Recipient, pub from_hopper: Recipient>, pub dns_failure_from_hopper: Recipient>, - pub add_return_route: Recipient, pub stream_shutdown_sub: Recipient, pub node_from_ui: Recipient, pub route_result_sub: Recipient, @@ -113,7 +104,6 @@ mod tests { recorder, ExpiredCoresPackage ), - add_return_route: recipient!(recorder, AddReturnRouteMessage), stream_shutdown_sub: recipient!(recorder, StreamShutdownMsg), node_from_ui: recipient!(recorder, NodeFromUiMessage), route_result_sub: recipient!(recorder, AddRouteResultMessage), diff --git a/node/src/sub_lib/route.rs b/node/src/sub_lib/route.rs index 6d9b52604..05bcc15ad 100644 --- a/node/src/sub_lib/route.rs +++ b/node/src/sub_lib/route.rs @@ -1,5 +1,4 @@ // Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved. -use crate::sub_lib::cryptde::encodex; use crate::sub_lib::cryptde::CryptDE; use crate::sub_lib::cryptde::CryptData; use crate::sub_lib::cryptde::PublicKey; @@ -33,7 +32,6 @@ impl Route { cryptde, None, None, - None, ) } @@ -48,7 +46,6 @@ impl Route { None, cryptde, consuming_wallet, - None, contract_address, ) } @@ -56,9 +53,8 @@ impl Route { pub fn round_trip( route_segment_over: RouteSegment, route_segment_back: RouteSegment, - cryptde: &dyn CryptDE, // Must be the CryptDE of the originating Node: used to encrypt return_route_id. + cryptde: &dyn CryptDE, // Doesn't matter which CryptDE: only used for encoding. consuming_wallet: Option, - return_route_id: u32, contract_address: Option
, ) -> Result { Self::construct( @@ -66,22 +62,10 @@ impl Route { Some(route_segment_back), cryptde, consuming_wallet, - Some(return_route_id), contract_address, ) } - pub fn id(&self, cryptde: &dyn CryptDE) -> Result { - if let Some(first) = self.hops.first() { - match decodex(cryptde, first) { - Ok(n) => Ok(n), - Err(e) => Err(format!("{:?}", e)), - } - } else { - Err("Response route did not contain a return route ID".to_string()) - } - } - // This cryptde must be the CryptDE of the next hop to come off the Route. pub fn next_hop(&self, cryptde: &dyn CryptDE) -> Result { match self.hops.first() { @@ -132,15 +116,7 @@ impl Route { last_cryptde.public_key(), live_hop ), - Err(outside) => match decodex::(last_cryptde, &last_hop_enc) { - Ok(return_route_id) => format!( - "{}\nEncrypted with {:?}: Return Route ID: {}\n", - most_strings, - last_cryptde.public_key(), - return_route_id - ), - Err(inside) => format!("{}\nError: {:?} / {:?}", most_strings, outside, inside), - }, + Err(error) => format!("{}\nError: {:?}", most_strings, error), } } @@ -149,7 +125,6 @@ impl Route { back: Option, cryptde: &dyn CryptDE, consuming_wallet: Option, - return_route_id_opt: Option, contract_address: Option
, ) -> Result { if let Some(error) = Route::validate_route_segments(&over, &back) { @@ -174,12 +149,7 @@ impl Route { contract_address, ); - Route::hops_to_route( - hops[0..].to_vec(), - &over.keys[0], - return_route_id_opt, - cryptde, - ) + Route::hops_to_route(hops[0..].to_vec(), &over.keys[0], cryptde) } fn over_segment<'a>( @@ -296,7 +266,6 @@ impl Route { fn hops_to_route( hops: Vec, top_hop_key: &PublicKey, - return_route_id_opt: Option, cryptde: &dyn CryptDE, ) -> Result { let mut hops_enc: Vec = Vec::new(); @@ -308,17 +277,8 @@ impl Route { }); hop_key = &data_hop.public_key; } - if let Some(return_route_id) = return_route_id_opt { - let return_route_id_enc = Self::encrypt_return_route_id(return_route_id, cryptde); - hops_enc.push(return_route_id_enc); - } Ok(Route { hops: hops_enc }) } - - fn encrypt_return_route_id(return_route_id: u32, cryptde: &dyn CryptDE) -> CryptData { - encodex(cryptde, cryptde.public_key(), &return_route_id) - .expect("Internal error encrypting u32 return_route_id") - } } pub struct RouteSegment { @@ -365,66 +325,6 @@ mod tests { static ref CRYPTDE_PAIR: CryptDEPair = CryptDEPair::null(); } - #[test] - fn id_decodes_return_route_id() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); - - let subject = Route { - hops: vec![Route::encrypt_return_route_id(42, cryptde)], - }; - - assert_eq!(subject.id(cryptde), Ok(42)); - } - - #[test] - fn id_returns_empty_route_error_when_the_route_is_empty() { - let cryptde = CRYPTDE_PAIR.main.as_ref(); - - let subject = Route { hops: vec![] }; - - assert_eq!( - subject.id(cryptde), - Err("Response route did not contain a return route ID".to_string()) - ); - } - - #[test] - #[should_panic(expected = "Could not decrypt with ebe5f9a0e2 data beginning with ebe5f9a0e1")] - fn id_returns_error_when_the_id_fails_to_decrypt() { - let cryptde1 = CryptDENull::from(&PublicKey::new(b"key a"), TEST_DEFAULT_CHAIN); - let cryptde2 = CryptDENull::from(&PublicKey::new(b"key b"), TEST_DEFAULT_CHAIN); - let subject = Route { - hops: vec![Route::encrypt_return_route_id(42, &cryptde1)], - }; - - let _ = subject.id(&cryptde2); - } - - #[test] - fn route_segments_are_represented_in_base64_by_debug() { - let public_key_data_1: Vec = vec![12, 34, 56, 78, 90]; - let public_key_data_2: Vec = vec![34, 56, 78, 90, 12]; - let public_key_data_3: Vec = vec![56, 78, 90, 12, 34]; - let subject = RouteSegment::new( - vec![ - &PublicKey::new(public_key_data_1.as_slice()), - &PublicKey::new(public_key_data_2.as_slice()), - &PublicKey::new(public_key_data_3.as_slice()), - ], - Component::ProxyClient, - ); - - let result = format!("{:?}", subject); - - let base64_1 = base64::encode_config(&public_key_data_1, base64::STANDARD_NO_PAD); - let base64_2 = base64::encode_config(&public_key_data_2, base64::STANDARD_NO_PAD); - let base64_3 = base64::encode_config(&public_key_data_3, base64::STANDARD_NO_PAD); - assert_eq!( - result, - format!("{} -> {} -> {} : ProxyClient", base64_1, base64_2, base64_3) - ); - } - #[test] fn construct_does_not_like_route_segments_with_too_few_keys() { let cryptde = CRYPTDE_PAIR.main.as_ref(); @@ -458,7 +358,6 @@ mod tests { RouteSegment::new(vec![&c_key, &d_key], Component::ProxyServer), cryptde, Some(paying_wallet.clone()), - 0, Some(TEST_DEFAULT_CHAIN.rec().contract), ) .err() @@ -502,7 +401,6 @@ mod tests { let f_key = PublicKey::new(&[70, 70, 70]); let cryptde = CRYPTDE_PAIR.main.as_ref(); let paying_wallet = make_paying_wallet(b"wallet"); - let return_route_id = 4321; let contract_address = TEST_DEFAULT_CHAIN.rec().contract; let subject = Route::round_trip( @@ -510,7 +408,6 @@ mod tests { RouteSegment::new(vec![&d_key, &e_key, &f_key, &a_key], Component::ProxyServer), cryptde, Some(paying_wallet.clone()), - return_route_id, Some(contract_address.clone()), ) .unwrap(); @@ -599,12 +496,6 @@ mod tests { .unwrap(), "seventh hop" ); - - assert_eq!( - subject.hops[7], - Route::encrypt_return_route_id(return_route_id, cryptde), - "eighth hop" - ); } #[test] @@ -783,7 +674,6 @@ mod tests { RouteSegment::new(vec![&key2, &key1], Component::ProxyServer), cryptde, Some(paying_wallet), - 1234, Some(TEST_DEFAULT_CHAIN.rec().contract), ) .unwrap(); @@ -832,13 +722,13 @@ Encrypted with 0x03040506: LiveHop { public_key: 0x, payer: Some(Payer { wallet: let key1 = PublicKey::new(&[1, 2, 3, 4]); let key2 = PublicKey::new(&[2, 3, 4, 5]); let key3 = PublicKey::new(&[3, 4, 5, 6]); + let cryptde = CryptDENull::from(&key1, TEST_DEFAULT_CHAIN); let paying_wallet = make_paying_wallet(b"wallet"); let subject = Route::round_trip( RouteSegment::new(vec![&key1, &key2, &key3], Component::ProxyClient), RouteSegment::new(vec![&key3, &key2, &key1], Component::ProxyServer), - &CryptDENull::from(&key1, TEST_DEFAULT_CHAIN), + &cryptde, Some(paying_wallet), - 1234, Some(TEST_DEFAULT_CHAIN.rec().contract), ) .unwrap(); @@ -861,7 +751,6 @@ Encrypted with 0x02030405: LiveHop { public_key: 0x03040506, payer: Some(Payer { Encrypted with 0x03040506: LiveHop { public_key: 0x02030405, payer: Some(Payer { wallet: Wallet { kind: Address(0x71d0fc7d1c570b1ed786382b551a09391c91e33d) }, proof: Signature { v: 1, r: "8649b8f6db6232cb1e4f1f04786ad4ef33488c968e64bec74ecd893d6d05c1b9", s: "8649b8f6db6232cb1e4f1f04786ad4ef33488c968e64bec74ecd893d6d05c1b9" } }), component: ProxyClient } Encrypted with 0x02030405: LiveHop { public_key: 0x01020304, payer: Some(Payer { wallet: Wallet { kind: Address(0x71d0fc7d1c570b1ed786382b551a09391c91e33d) }, proof: Signature { v: 0, r: "4324a40295bb36ef2b927fb24250fe42397a57b861ea152bbbe4f84150d4ff5a", s: "4324a40295bb36ef2b927fb24250fe42397a57b861ea152bbbe4f84150d4ff5a" } }), component: Hopper } Encrypted with 0x01020304: LiveHop { public_key: 0x, payer: Some(Payer { wallet: Wallet { kind: Address(0x71d0fc7d1c570b1ed786382b551a09391c91e33d) }, proof: Signature { v: 0, r: "3e3a92d7284c2c2ff7119e9f7a7e183b062a335a598e965a47c36a2f288b6f8d", s: "3e3a92d7284c2c2ff7119e9f7a7e183b062a335a598e965a47c36a2f288b6f8d" } }), component: ProxyServer } -Encrypted with 0x01020304: Return Route ID: 1234 "# ) ); diff --git a/node/src/sub_lib/sequence_buffer.rs b/node/src/sub_lib/sequence_buffer.rs index c73dbc021..2cb101c41 100644 --- a/node/src/sub_lib/sequence_buffer.rs +++ b/node/src/sub_lib/sequence_buffer.rs @@ -38,7 +38,7 @@ impl<'a> From<&'a TransmitDataMsg> for SequencedPacket { fn from(tdm: &'a TransmitDataMsg) -> Self { SequencedPacket::new( tdm.data.clone(), - tdm.sequence_number.unwrap_or(0), + tdm.sequence_number_opt.unwrap_or(0), tdm.last_data, ) } @@ -253,7 +253,7 @@ mod tests { endpoint: Endpoint::Socket(SocketAddr::from_str("1.2.3.4:80").unwrap()), last_data: true, data: vec![1, 4, 5, 9], - sequence_number: None, + sequence_number_opt: None, }; let result = SequencedPacket::from(&tdm); @@ -267,7 +267,7 @@ mod tests { endpoint: Endpoint::Socket(SocketAddr::from_str("1.2.3.4:80").unwrap()), last_data: true, data: vec![1, 4, 5, 9], - sequence_number: Some(1), + sequence_number_opt: Some(1), }; let result = SequencedPacket::from(&tdm); @@ -280,7 +280,7 @@ mod tests { endpoint: Endpoint::Socket(SocketAddr::from_str("1.2.3.4:80").unwrap()), last_data: false, data: vec![4, 2, 5, 67], - sequence_number: Some(4), + sequence_number_opt: Some(4), }; let result = SequencedPacket::from(&tdm); diff --git a/node/src/sub_lib/stream_handler_pool.rs b/node/src/sub_lib/stream_handler_pool.rs index fcb0f2e64..92ade9931 100644 --- a/node/src/sub_lib/stream_handler_pool.rs +++ b/node/src/sub_lib/stream_handler_pool.rs @@ -8,7 +8,7 @@ use actix::Message; pub struct TransmitDataMsg { pub endpoint: Endpoint, pub last_data: bool, - pub sequence_number: Option, // Some implies clear data; None implies clandestine. + pub sequence_number_opt: Option, // Some implies clear data; None implies clandestine. pub data: Vec, } diff --git a/node/src/sub_lib/stream_key.rs b/node/src/sub_lib/stream_key.rs index bca70efe4..8e6629ab5 100644 --- a/node/src/sub_lib/stream_key.rs +++ b/node/src/sub_lib/stream_key.rs @@ -93,13 +93,22 @@ impl StreamKey { hash: hash.digest().bytes(), } } + + #[cfg(test)] + pub fn from_bytes(bytes: &[u8]) -> StreamKey { + let mut hash = [0xA; sha1::DIGEST_LENGTH]; + for i in 0..std::cmp::min(sha1::DIGEST_LENGTH, bytes.len()) { + hash[i] = bytes[i]; + } + StreamKey { hash } + } } impl StreamKey { pub fn make_meaningless_stream_key() -> StreamKey { - StreamKey { - hash: [0; sha1::DIGEST_LENGTH], - } + let mut bytes = [0; sha1::DIGEST_LENGTH]; + randombytes_into(&mut bytes); + StreamKey { hash: bytes } } pub fn make_meaningful_stream_key(phrase: &str) -> StreamKey { diff --git a/node/src/sub_lib/ttl_hashmap.rs b/node/src/sub_lib/ttl_hashmap.rs index faa79b68a..4782e78a3 100644 --- a/node/src/sub_lib/ttl_hashmap.rs +++ b/node/src/sub_lib/ttl_hashmap.rs @@ -8,6 +8,7 @@ use std::rc::Rc; use std::time::Duration; use std::time::Instant; +#[allow(clippy::type_complexity)] pub struct TtlHashMap where K: Hash + Clone, @@ -15,6 +16,7 @@ where last_check: RefCell, data: RefCell, Instant)>>, ttl: Duration, + retire_closure: Box bool>, } impl TtlHashMap @@ -27,6 +29,19 @@ where last_check: RefCell::new(Instant::now()), data: RefCell::new(HashMap::new()), ttl, + retire_closure: Box::new(|_, _| true), + } + } + + pub fn new_with_retire(ttl: Duration, retire_closure: F) -> TtlHashMap + where + F: 'static + Fn(&K, &V) -> bool, + { + TtlHashMap { + last_check: RefCell::new(Instant::now()), + data: RefCell::new(HashMap::new()), + ttl, + retire_closure: Box::new(retire_closure), } } @@ -54,6 +69,12 @@ where } } + pub fn remove(&self, key: &K) -> Option> { + self.remove_expired_entries(); + + self.data.borrow_mut().remove(key).map(|(result, _)| result) + } + fn remove_expired_entries(&self) { let now = Instant::now(); @@ -73,8 +94,16 @@ where .collect() }; + let mut data = self.data.borrow_mut(); expired.iter().for_each(|key| { - self.data.borrow_mut().remove(key); + match data.remove(key) { + Some((value, _)) => { + if !(self.retire_closure)(key, value.as_ref()) { + data.insert(key.clone(), (value, now)); + } + } + None => (), // already removed + } }); } } @@ -82,26 +111,70 @@ where #[cfg(test)] mod tests { use super::*; + use std::sync::{Arc, Mutex}; use std::thread; #[test] fn new_sets_ttl() { let subject = TtlHashMap::::new(Duration::from_millis(1000)); - assert_eq!(subject.ttl, Duration::from_millis(1000)); + let result = subject.ttl; + + assert_eq!(result, Duration::from_millis(1000)); } #[test] - fn remove_returns_none_for_entry_that_was_never_inserted() { + fn get_returns_none_for_entry_that_was_never_inserted() { let subject = TtlHashMap::::new(Duration::from_millis(1000)); - assert_eq!(subject.get(&11u32), None); + let result = subject.get(&11u32); + + assert_eq!(result, None); assert_eq!(subject.ttl(), Duration::from_millis(1000)); } #[test] - fn ttl_hashmap_does_not_remove_entry_before_it_is_expired() { + fn remove_returns_none_if_no_such_entry_exists() { + let subject = TtlHashMap::::new(Duration::from_millis(1000)); + + let result = subject.remove(&11u32); + + assert_eq!(result, None); + } + + #[test] + fn remove_returns_existing_entry_and_removes() { + let mut subject = TtlHashMap::::new(Duration::from_millis(1000)); + subject.insert(11u32, 42u32); + + let before_result = subject.remove(&11u32); + let after_result = subject.remove(&11u32); + + assert_eq!(before_result, Some(Rc::new(42u32))); + assert_eq!(after_result, None); + } + + #[test] + fn ttl_hashmap_remove_removes_expired_entry() { let mut subject = TtlHashMap::new(Duration::from_millis(10)); + subject.insert(42u32, "Hello"); + thread::sleep(Duration::from_millis(20)); + + let result = subject.remove(&11u32); // nonexistent key + + assert_eq!(result, None); + // Low-level get, because high-level get would remove it if .remove() didn't + assert_eq!(subject.data.borrow().get(&42u32), None); + } + + #[test] + fn ttl_hashmap_does_not_remove_entry_before_it_is_expired() { + let retire_closure_has_run = Arc::new(Mutex::new(false)); + let retire_closure_has_run_inner = retire_closure_has_run.clone(); + let mut subject = TtlHashMap::new_with_retire(Duration::from_millis(10), move |_, _| { + *(retire_closure_has_run_inner.lock().unwrap()) = true; + true + }); subject.insert(42u32, "Hello"); subject.insert(24u32, "World"); @@ -109,25 +182,52 @@ mod tests { assert_eq!(subject.get(&42u32).unwrap().as_ref(), &"Hello"); assert_eq!(subject.get(&24u32).unwrap().as_ref(), &"World"); assert_eq!(subject.ttl(), Duration::from_millis(10)); + assert_eq!(*retire_closure_has_run.lock().unwrap(), false); } #[test] fn ttl_hashmap_get_removes_expired_entry() { - let mut subject = TtlHashMap::new(Duration::from_millis(10)); - + let retire_closure_has_run = Arc::new(Mutex::new(false)); + let retire_closure_has_run_inner = retire_closure_has_run.clone(); + let mut subject = TtlHashMap::new_with_retire(Duration::from_millis(10), move |_, _| { + *(retire_closure_has_run_inner.lock().unwrap()) = true; + true + }); subject.insert(42u32, "Hello"); + thread::sleep(Duration::from_millis(20)); + + let result = subject.get(&42u32); + + assert_eq!(result, None); + assert_eq!(*retire_closure_has_run.lock().unwrap(), true); + } + #[test] + fn ttl_hashmap_get_does_not_remove_expired_entry_if_closure_returns_false() { + let retire_closure_has_run = Arc::new(Mutex::new(false)); + let retire_closure_has_run_inner = retire_closure_has_run.clone(); + let mut subject = TtlHashMap::new_with_retire(Duration::from_millis(10), move |_, _| { + *(retire_closure_has_run_inner.lock().unwrap()) = true; + false + }); + subject.insert(42u32, "Hello"); thread::sleep(Duration::from_millis(20)); - assert_eq!(subject.get(&42u32), None); + let result = subject.get(&42u32); + + assert_eq!(result, Some(Rc::new("Hello"))); + assert_eq!(*retire_closure_has_run.lock().unwrap(), true); } #[test] fn ttl_hashmap_insert_removes_expired_entry() { - let mut subject = TtlHashMap::new(Duration::from_millis(10)); - + let retire_closure_has_run = Arc::new(Mutex::new(false)); + let retire_closure_has_run_inner = retire_closure_has_run.clone(); + let mut subject = TtlHashMap::new_with_retire(Duration::from_millis(10), move |_, _| { + *(retire_closure_has_run_inner.lock().unwrap()) = true; + true + }); subject.insert(42u32, "Hello"); - thread::sleep(Duration::from_millis(20)); subject.insert(24u32, "World"); @@ -137,6 +237,7 @@ mod tests { subject.data.borrow().get(&24u32).unwrap().0.as_ref(), &"World" ); + assert_eq!(*retire_closure_has_run.lock().unwrap(), true); } #[test] diff --git a/node/src/test_utils/database_utils.rs b/node/src/test_utils/database_utils.rs index a2b6d9ee1..c100cec44 100644 --- a/node/src/test_utils/database_utils.rs +++ b/node/src/test_utils/database_utils.rs @@ -7,6 +7,10 @@ use crate::database::db_initializer::ExternalData; use crate::database::rusqlite_wrappers::ConnectionWrapper; use crate::database::db_migrations::db_migrator::DbMigrator; +use crate::db_config::persistent_configuration::{ + PersistentConfiguration, PersistentConfigurationFactory, +}; +use crate::test_utils::persistent_configuration_mock::PersistentConfigurationMock; use masq_lib::logger::Logger; use masq_lib::test_utils::utils::TEST_DEFAULT_CHAIN; use masq_lib::utils::{to_string, NeighborhoodModeLight}; @@ -308,3 +312,26 @@ pub fn make_external_data() -> ExternalData { db_password_opt: None, } } + +pub struct PersistentConfigurationFactoryMock { + mock_opt: RefCell>, +} + +impl PersistentConfigurationFactory for PersistentConfigurationFactoryMock { + fn make(&self) -> Box { + Box::new( + self.mock_opt + .borrow_mut() + .take() + .expect("PersistentConfigurationFactoryTest already used"), + ) + } +} + +impl PersistentConfigurationFactoryMock { + pub fn new(mock: PersistentConfigurationMock) -> Self { + Self { + mock_opt: RefCell::new(Some(mock)), + } + } +} diff --git a/node/src/test_utils/mod.rs b/node/src/test_utils/mod.rs index eeb2c9560..6e4f0adb5 100644 --- a/node/src/test_utils/mod.rs +++ b/node/src/test_utils/mod.rs @@ -23,7 +23,6 @@ use crate::blockchain::bip32::Bip32EncryptionKeyProvider; use crate::blockchain::payer::Payer; use crate::bootstrapper::CryptDEPair; use crate::sub_lib::cryptde::CryptDE; -use crate::sub_lib::cryptde::CryptData; use crate::sub_lib::cryptde::PlainData; use crate::sub_lib::cryptde::PublicKey; use crate::sub_lib::dispatcher::Component; @@ -54,8 +53,10 @@ use std::iter::repeat; use std::net::{Shutdown, TcpStream}; use crate::sub_lib::hopper::MessageType; +use crate::sub_lib::host::Host; use crate::sub_lib::proxy_client::DnsResolveFailure_0v1; use crate::sub_lib::stream_key::StreamKey; +use masq_lib::constants::{HTTP_PORT, TLS_PORT}; use std::str::FromStr; use std::sync::{Arc, Mutex}; use std::thread; @@ -197,25 +198,26 @@ pub fn make_meaningless_wallet_private_key() -> PlainData { } // TODO: The three functions below should use only one argument, cryptde -pub fn route_to_proxy_client(main_key: &PublicKey, main_cryptde: &dyn CryptDE) -> Route { +pub fn route_to_proxy_client(main_key: &PublicKey, main_cryptde: &dyn CryptDE, tls: bool) -> Route { shift_one_hop( - zero_hop_route_response(main_key, main_cryptde).route, + zero_hop_route_response(main_key, main_cryptde, tls).route, main_cryptde, ) } -pub fn route_from_proxy_client(key: &PublicKey, cryptde: &dyn CryptDE) -> Route { +pub fn route_from_proxy_client(key: &PublicKey, cryptde: &dyn CryptDE, tls: bool) -> Route { // Happens to be the same - route_to_proxy_client(key, cryptde) + route_to_proxy_client(key, cryptde, tls) } -pub fn route_to_proxy_server(key: &PublicKey, cryptde: &dyn CryptDE) -> Route { - shift_one_hop(route_from_proxy_client(key, cryptde), cryptde) +pub fn route_to_proxy_server(key: &PublicKey, cryptde: &dyn CryptDE, tls: bool) -> Route { + shift_one_hop(route_from_proxy_client(key, cryptde, tls), cryptde) } pub fn zero_hop_route_response( public_key: &PublicKey, cryptde: &dyn CryptDE, + tls: bool, ) -> RouteQueryResponse { RouteQueryResponse { route: Route::round_trip( @@ -223,15 +225,14 @@ pub fn zero_hop_route_response( RouteSegment::new(vec![public_key, public_key], Component::ProxyServer), cryptde, None, - 0, None, ) .unwrap(), expected_services: ExpectedServices::RoundTrip( vec![ExpectedService::Nothing, ExpectedService::Nothing], vec![ExpectedService::Nothing, ExpectedService::Nothing], - 0, ), + host: Host::new("booga.com", if tls { TLS_PORT } else { HTTP_PORT }), } } @@ -240,13 +241,6 @@ fn shift_one_hop(mut route: Route, cryptde: &dyn CryptDE) -> Route { route } -pub fn encrypt_return_route_id(return_route_id: u32, cryptde: &dyn CryptDE) -> CryptData { - let return_route_id_ser = serde_cbor::ser::to_vec(&return_route_id).unwrap(); - cryptde - .encode(cryptde.public_key(), &PlainData::from(return_route_id_ser)) - .unwrap() -} - pub fn make_garbage_data(bytes: usize) -> Vec { let mut data = vec![0; bytes]; rand::thread_rng().fill_bytes(&mut data); @@ -431,8 +425,8 @@ pub fn read_until_timeout(stream: &mut dyn Read) -> Vec { response } -pub fn make_meaningless_message_type() -> MessageType { - DnsResolveFailure_0v1::new(StreamKey::make_meaningless_stream_key()).into() +pub fn make_meaningless_message_type(stream_key: StreamKey) -> MessageType { + DnsResolveFailure_0v1::new(stream_key).into() } pub fn handle_connection_error(stream: TcpStream) { @@ -509,7 +503,9 @@ pub mod unshared_test_utils { use crate::node_test_utils::DirsWrapperMock; use crate::sub_lib::accountant::{PaymentThresholds, ScanIntervals}; use crate::sub_lib::cryptde::CryptDE; - use crate::sub_lib::neighborhood::{ConnectionProgressMessage, DEFAULT_RATE_PACK}; + use crate::sub_lib::neighborhood::{ + ConnectionProgressMessage, RatePack, RatePackLimits, DEFAULT_RATE_PACK, + }; use crate::sub_lib::proxy_client::ClientResponsePayload_0v1; use crate::sub_lib::proxy_server::{ClientRequestPayload_0v1, ProxyProtocol}; use crate::sub_lib::sequence_buffer::SequencedPacket; @@ -585,13 +581,13 @@ pub mod unshared_test_utils { Either::Left((message_start, message_end)) => { assert!( panic_message_str.contains(message_start), - "We expected this message {} to start with {}", + "We expected this message '{}' to start with '{}'", panic_message_str, message_start ); assert!( panic_message_str.ends_with(message_end), - "We expected this message {} to end with {}", + "We expected this message '{}' to end with '{}'", panic_message_str, message_end ); @@ -654,6 +650,10 @@ pub mod unshared_test_utils { } else { config }; + let config = config.rate_pack_limits_result(Ok(RatePackLimits::new( + RatePack::new(u64::MIN, u64::MIN, u64::MIN, u64::MIN), + RatePack::new(u64::MAX, u64::MAX, u64::MAX, u64::MAX), + ))); config } @@ -717,7 +717,7 @@ pub mod unshared_test_utils { ClientRequestPayload_0v1 { stream_key: StreamKey::make_meaningful_stream_key("request"), sequenced_packet: SequencedPacket::new(make_garbage_data(bytes), 0, true), - target_hostname: Some("www.example.com".to_string()), + target_hostname: "www.example.com".to_string(), target_port: HTTP_PORT, protocol: ProxyProtocol::HTTP, originator_public_key: cryptde.public_key().clone(), @@ -1325,7 +1325,7 @@ mod tests { let cryptde = CRYPTDE_PAIR.main.as_ref(); let key = cryptde.public_key(); - let subject = zero_hop_route_response(&key, cryptde); + let subject = zero_hop_route_response(&key, cryptde, false); assert_eq!( subject.route.hops, @@ -1339,7 +1339,6 @@ mod tests { LiveHop::new(&PublicKey::new(b""), None, Component::ProxyServer) .encode(&key, cryptde) .unwrap(), - encrypt_return_route_id(0, cryptde), ) ); assert_eq!( @@ -1347,7 +1346,6 @@ mod tests { ExpectedServices::RoundTrip( vec![ExpectedService::Nothing, ExpectedService::Nothing,], vec![ExpectedService::Nothing, ExpectedService::Nothing,], - 0 ) ); } @@ -1357,7 +1355,7 @@ mod tests { let cryptde = CRYPTDE_PAIR.main.as_ref(); let key = cryptde.public_key(); - let subject = route_to_proxy_client(&key, cryptde); + let subject = route_to_proxy_client(&key, cryptde, false); let mut garbage_can: Vec = iter::repeat(0u8).take(96).collect(); cryptde.random(&mut garbage_can[..]); @@ -1370,7 +1368,6 @@ mod tests { LiveHop::new(&PublicKey::new(b""), None, Component::ProxyServer) .encode(&key, cryptde) .unwrap(), - encrypt_return_route_id(0, cryptde), CryptData::new(&garbage_can[..]) ) ); @@ -1381,7 +1378,7 @@ mod tests { let cryptde = CRYPTDE_PAIR.main.as_ref(); let key = cryptde.public_key(); - let subject = route_from_proxy_client(&key, cryptde); + let subject = route_from_proxy_client(&key, cryptde, false); let mut garbage_can: Vec = iter::repeat(0u8).take(96).collect(); cryptde.random(&mut garbage_can[..]); @@ -1394,7 +1391,6 @@ mod tests { LiveHop::new(&PublicKey::new(b""), None, Component::ProxyServer) .encode(&key, cryptde) .unwrap(), - encrypt_return_route_id(0, cryptde), CryptData::new(&garbage_can[..]) ) ); @@ -1405,7 +1401,7 @@ mod tests { let cryptde = CRYPTDE_PAIR.main.as_ref(); let key = cryptde.public_key(); - let subject = route_to_proxy_server(&key, cryptde); + let subject = route_to_proxy_server(&key, cryptde, false); let mut first_garbage_can: Vec = iter::repeat(0u8).take(96).collect(); let mut second_garbage_can: Vec = iter::repeat(0u8).take(96).collect(); @@ -1417,7 +1413,6 @@ mod tests { LiveHop::new(&PublicKey::new(b""), None, Component::ProxyServer) .encode(&key, cryptde) .unwrap(), - encrypt_return_route_id(0, cryptde), CryptData::new(&first_garbage_can[..]), CryptData::new(&second_garbage_can[..]), ) diff --git a/node/src/test_utils/persistent_configuration_mock.rs b/node/src/test_utils/persistent_configuration_mock.rs index 138a1deb6..ac9264a8d 100644 --- a/node/src/test_utils/persistent_configuration_mock.rs +++ b/node/src/test_utils/persistent_configuration_mock.rs @@ -6,7 +6,7 @@ use crate::database::rusqlite_wrappers::TransactionSafeWrapper; use crate::db_config::persistent_configuration::{PersistentConfigError, PersistentConfiguration}; use crate::sub_lib::accountant::{PaymentThresholds, ScanIntervals}; use crate::sub_lib::cryptde::CryptDE; -use crate::sub_lib::neighborhood::{Hops, NodeDescriptor, RatePack}; +use crate::sub_lib::neighborhood::{Hops, NodeDescriptor, RatePack, RatePackLimits}; use crate::sub_lib::wallet::Wallet; use crate::test_utils::unshared_test_utils::arbitrary_id_stamp::ArbitraryIdStamp; use crate::{arbitrary_id_stamp_in_trait_impl, set_arbitrary_id_stamp_in_mock_impl}; @@ -76,6 +76,7 @@ pub struct PersistentConfigurationMock { set_payment_thresholds_params: Arc>>, set_payment_thresholds_results: RefCell>>, rate_pack_results: RefCell>>, + rate_pack_limits_results: RefCell>>, set_rate_pack_params: Arc>>, set_rate_pack_results: RefCell>>, scan_intervals_results: RefCell>>, @@ -152,6 +153,7 @@ impl Clone for PersistentConfigurationMock { set_payment_thresholds_params: self.set_payment_thresholds_params.clone(), set_payment_thresholds_results: self.set_payment_thresholds_results.clone(), rate_pack_results: self.rate_pack_results.clone(), + rate_pack_limits_results: self.rate_pack_limits_results.clone(), set_rate_pack_params: self.set_rate_pack_params.clone(), set_rate_pack_results: self.set_rate_pack_results.clone(), scan_intervals_results: self.scan_intervals_results.clone(), @@ -402,6 +404,10 @@ impl PersistentConfiguration for PersistentConfigurationMock { self.set_rate_pack_results.borrow_mut().remove(0) } + fn rate_pack_limits(&self) -> Result { + self.rate_pack_limits_results.borrow_mut().remove(0) + } + fn scan_intervals(&self) -> Result { self.scan_intervals_results.borrow_mut().remove(0) } @@ -759,6 +765,14 @@ impl PersistentConfigurationMock { self } + pub fn rate_pack_limits_result( + self, + result: Result, + ) -> Self { + self.rate_pack_limits_results.borrow_mut().push(result); + self + } + pub fn set_rate_pack_params(mut self, params: &Arc>>) -> Self { self.set_rate_pack_params = params.clone(); self diff --git a/node/src/test_utils/recorder.rs b/node/src/test_utils/recorder.rs index f52b1a0c8..299e9828a 100644 --- a/node/src/test_utils/recorder.rs +++ b/node/src/test_utils/recorder.rs @@ -39,10 +39,8 @@ use crate::sub_lib::peer_actors::PeerActors; use crate::sub_lib::peer_actors::{BindMessage, NewPublicIp, StartMessage}; use crate::sub_lib::proxy_client::{ClientResponsePayload_0v1, InboundServerData}; use crate::sub_lib::proxy_client::{DnsResolveFailure_0v1, ProxyClientSubs}; -use crate::sub_lib::proxy_server::{ - AddReturnRouteMessage, ClientRequestPayload_0v1, StreamKeyPurge, -}; use crate::sub_lib::proxy_server::{AddRouteResultMessage, ProxyServerSubs}; +use crate::sub_lib::proxy_server::{ClientRequestPayload_0v1, StreamKeyPurge}; use crate::sub_lib::stream_handler_pool::DispatcherNodeQueryResponse; use crate::sub_lib::stream_handler_pool::TransmitDataMsg; use crate::sub_lib::ui_gateway::UiGatewaySubs; @@ -127,7 +125,6 @@ macro_rules! recorder_message_handler_t_p { }; } -recorder_message_handler_t_m_p!(AddReturnRouteMessage); recorder_message_handler_t_m_p!(AddRouteResultMessage); recorder_message_handler_t_p!(AddStreamMsg); recorder_message_handler_t_m_p!(BindMessage); @@ -462,7 +459,6 @@ pub fn make_proxy_server_subs_from_recorder(addr: &Addr) -> ProxyServe from_dispatcher: recipient!(addr, InboundClientData), from_hopper: recipient!(addr, ExpiredCoresPackage), dns_failure_from_hopper: recipient!(addr, ExpiredCoresPackage), - add_return_route: recipient!(addr, AddReturnRouteMessage), stream_shutdown_sub: recipient!(addr, StreamShutdownMsg), node_from_ui: recipient!(addr, NodeFromUiMessage), route_result_sub: recipient!(addr, AddRouteResultMessage), @@ -641,8 +637,9 @@ impl PeerActorsBuilder { } // This must be called after System.new and before System.run. - // These addresses may be helpful for setting up the Counter Messages. - pub fn build_and_provide_addresses(self) -> (PeerActors, PeerActorAddrs) { + // + // The addresses may be helpful for setting up the Counter Messages. + pub fn build_with_addresses(self) -> (PeerActors, PeerActorAddrs) { let proxy_server_addr = self.proxy_server.start(); let dispatcher_addr = self.dispatcher.start(); let hopper_addr = self.hopper.start(); @@ -683,7 +680,7 @@ impl PeerActorsBuilder { // This must be called after System.new and before System.run pub fn build(self) -> PeerActors { - let (peer_actors, _) = self.build_and_provide_addresses(); + let (peer_actors, _) = self.build_with_addresses(); peer_actors } } diff --git a/node/tests/contract_test.rs b/node/tests/contract_test.rs index 42d6fce37..35b7e0f85 100644 --- a/node/tests/contract_test.rs +++ b/node/tests/contract_test.rs @@ -133,7 +133,10 @@ fn masq_erc20_contract_exists_on_polygon_mainnet_integration() { #[test] fn masq_erc20_contract_exists_on_ethereum_mainnet_integration() { - let blockchain_urls = vec!["https://mainnet.infura.io/v3/0ead23143b174f6983c76f69ddcf4026"]; + let blockchain_urls = vec![ + "https://eth.llamarpc.com", + "https://mainnet.infura.io/v3/0ead23143b174f6983c76f69ddcf4026", + ]; let chain = Chain::EthMainnet; let assertion_body = |url, chain| assert_contract_existence(url, chain, "MASQ", 18); @@ -207,7 +210,10 @@ fn assert_total_supply( #[test] fn max_token_supply_matches_corresponding_constant_integration() { - let blockchain_urls = vec!["https://mainnet.infura.io/v3/0ead23143b174f6983c76f69ddcf4026"]; + let blockchain_urls = vec![ + "https://eth.llamarpc.com", + "https://mainnet.infura.io/v3/0ead23143b174f6983c76f69ddcf4026", + ]; let chain = Chain::EthMainnet; let assertion_body = |url, chain| assert_total_supply(url, chain, MASQ_TOTAL_SUPPLY); diff --git a/port_exposer/Cargo.lock b/port_exposer/Cargo.lock index 1a65f5fc1..ee921a2ed 100644 --- a/port_exposer/Cargo.lock +++ b/port_exposer/Cargo.lock @@ -20,7 +20,7 @@ checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" [[package]] name = "port_exposer" -version = "0.9.0" +version = "0.9.1" dependencies = [ "default-net", ] diff --git a/port_exposer/Cargo.toml b/port_exposer/Cargo.toml index 703fa9813..02454f223 100644 --- a/port_exposer/Cargo.toml +++ b/port_exposer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "port_exposer" -version = "0.9.0" +version = "0.9.1" authors = ["Dan Wiebe ", "MASQ"] license = "GPL-3.0-only" copyright = "Copyright (c) 2019, MASQ (https://masq.ai) and/or its affiliates. All rights reserved." diff --git a/test_utilities/Cargo.toml b/test_utilities/Cargo.toml index 93191e691..b4107a19e 100644 --- a/test_utilities/Cargo.toml +++ b/test_utilities/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "test_utilities" -version = "0.1.0" +version = "0.9.1" edition = "2021" authors = ["Dan Wiebe ", "MASQ"] license = "GPL-3.0-only"