Skip to content

Commit

Permalink
HTTP/AddressBook: major refactoring + download impl
Browse files Browse the repository at this point in the history
Major:
- Implementation improvements
  * HTTP: download impl
  * HTTP: URI handling impl
  * HTTP: response header handling for clearnet
  * AddressBook: unhook in-net HTTP impl (now in class HTTP)
  * Add documentation

Medium:
- If default subscription file does not exist,
  create/save after downloading from publisher

Minor:
- Adjust related reseed and tunnel code for class HTTP
- Update variables names to accurately reflect purpose
- Update log messages for accuracy
- Update address book flow narrative
- Update/improve HTTP unit-test

References:
- monero-project#305 (parent ticket)
- monero-project#168 and monero-project#155 (both in terms of HTTP/S)
- monero-project#154 and monero-project#48 (we now have a multi-purpose HTTP/S download impl)

Notes:
- My apologies for not keeping this commit atomic.
  Bigger design work can be harder to keep atomic
  (especially if one wants new features to work before committing)
- This commit has been successfully tested and all unit-tests pass
anonimal committed Sep 5, 2016

Unverified

The email in this signature doesn’t match the committer email.
1 parent 7458653 commit add429b
Showing 10 changed files with 647 additions and 456 deletions.
399 changes: 140 additions & 259 deletions src/client/address_book.cc

Large diffs are not rendered by default.

33 changes: 13 additions & 20 deletions src/client/address_book.h
Original file line number Diff line number Diff line change
@@ -50,12 +50,13 @@
#include "destination.h"
#include "router_context.h"
#include "util/base64.h"
#include "util/http.h"
#include "util/log.h"

namespace i2p {
namespace client {

/// @enum SubscriptionTimeout
/// @enum SubscriberTimeout
/// @brief Constants used for timeout intervals when fetching subscriptions
/// @notes Scoped to prevent namespace pollution (otherwise, purely stylistic)
enum struct SubscriberTimeout : std::uint16_t {
@@ -64,9 +65,6 @@ enum struct SubscriberTimeout : std::uint16_t {
InitialRetry = 1,
ContinuousUpdate = 720, // 12 hours
ContinuousRetry = 5,
// Seconds
Request = 60,
Receive = 30,
};

class AddressBookSubscriber;
@@ -78,9 +76,8 @@ class AddressBook {
AddressBook()
: m_SharedLocalDestination(nullptr),
m_Storage(nullptr),
m_DefaultSubscriber(nullptr),
m_SubscriberUpdateTimer(nullptr),
m_HostsAreLoaded(false),
m_SubscriptionIsLoaded(false),
m_SubscriberIsDownloading(false) {}

/// @brief Stops address book implementation
@@ -195,8 +192,8 @@ class AddressBook {
/// @return Const reference to publishers filename
/// @notes A publishers file holds a list of publisher addresses
/// of whom publish 'subscriptions' that contain a list of hosts to .b32.i2p
const std::string& GetDefaultPublishersFilename() {
return AddressBookDefaults.at("PublisherFilename");
const std::string& GetPublishersFilename() {
return AddressBookDefaults.at("PublishersFilename");
}

/// @brief Gets default publishers URI
@@ -209,7 +206,7 @@ class AddressBook {
/// @brief Gets default subscription filename
/// @return Const reference to subscription filename
/// @notes Filename used by publishers when they publish a 'subscription'
const std::string& GetDefaultSubscriptionFilename() {
const std::string& GetSubscriptionFilename() {
return AddressBookDefaults.at("SubscriptionFilename");
}

@@ -246,19 +243,14 @@ class AddressBook {
/// @brief Vector of unique pointers to respective subscriber implementation
std::vector<std::unique_ptr<AddressBookSubscriber>> m_Subscribers;

/// @var m_DefaultSubscriber
/// @brief Unique pointer to address book subscriber implementation
/// @notes Used if we don't have any publisher addresses on file
std::unique_ptr<AddressBookSubscriber> m_DefaultSubscriber;

/// @var m_SubscriberUpdateTimer
/// @brief Unique pointer to Boost.Asio deadline_timer
/// @details Handles all timer-related needs for subscription fetching
std::unique_ptr<boost::asio::deadline_timer> m_SubscriberUpdateTimer;

/// @var m_HostsAreLoaded
/// @var m_SubscriptionIsLoaded
/// @brief Are hosts loaded into memory?
std::atomic<bool> m_HostsAreLoaded;
std::atomic<bool> m_SubscriptionIsLoaded;

/// @var m_SubscriberIsDownloading
/// @brief Are subscriptions in the process of being downloaded?
@@ -274,7 +266,7 @@ class AddressBookSubscriber {
AddressBook& book,
const std::string& uri)
: m_Book(book),
m_URI(uri) {}
m_HTTP(uri) {}

/// @brief Instantiates thread that fetches in-net subscriptions
void DownloadSubscription();
@@ -284,12 +276,13 @@ class AddressBookSubscriber {
/// @warning Must be run in separate thread
void DownloadSubscriptionImpl();

private:
/// @var m_Book
/// @brief Reference to address book implementation
AddressBook& m_Book;
// Used for HTTP request // TODO(anonimal): remove when refactored with cpp-netlib
std::string m_URI, m_Etag, m_LastModified;

/// @var m_HTTP
/// @brief HTTP instance for subscribing to publisher
i2p::util::http::HTTP m_HTTP;
};

} // namespace client
12 changes: 7 additions & 5 deletions src/client/address_book_storage.cc
Original file line number Diff line number Diff line change
@@ -105,7 +105,7 @@ void AddressBookFilesystemStorage::RemoveAddress(
std::size_t AddressBookFilesystemStorage::Load(
std::map<std::string, i2p::data::IdentHash>& addresses) {
std::size_t num = 0;
auto filename = GetAddressesFilename();
auto filename = GetAddressesFilenamePath();
std::ifstream file(filename, std::ofstream::in);
if (file.is_open()) {
addresses.clear();
@@ -120,7 +120,7 @@ std::size_t AddressBookFilesystemStorage::Load(
std::string addr = host.substr(pos);
i2p::data::IdentHash ident;
ident.FromBase32(addr);
addresses[name] = ident;
addresses[name] = ident; // TODO(anonimal): bounds checking
num++;
}
}
@@ -136,16 +136,18 @@ std::size_t AddressBookFilesystemStorage::Load(
std::size_t AddressBookFilesystemStorage::Save(
const std::map<std::string, i2p::data::IdentHash>& addresses) {
std::size_t num = 0;
auto filename = GetAddressesFilename();
auto filename = GetAddressesFilenamePath();
std::ofstream file(filename, std::ofstream::out);
if (file.is_open()) {
for (auto it : addresses) {
file << it.first << "," << it.second.ToBase32() << std::endl;
num++;
}
LogPrint(eLogInfo, "AddressBookFilesystemStorage: ", num, " addresses saved");
LogPrint(eLogInfo,
"AddressBookFilesystemStorage: ", num, " addresses saved");
} else {
LogPrint(eLogError, "AddressBookFilesystemStorage: can't open file ", filename);
LogPrint(eLogError,
"AddressBookFilesystemStorage: can't open file ", filename);
}
return num;
}
11 changes: 8 additions & 3 deletions src/client/address_book_storage.h
Original file line number Diff line number Diff line change
@@ -52,6 +52,7 @@
namespace i2p {
namespace client {

// TODO(anonimal): remove this map in favor of a struct for defaults
/// @var AddressBookDefaults
/// @brief Default string constants used throughout address book
const std::unordered_map<std::string, std::string> AddressBookDefaults = {
@@ -60,9 +61,11 @@ const std::unordered_map<std::string, std::string> AddressBookDefaults = {
// TODO(anonimal): "addresses" is confusing and so is its real purpose
// (currently only used to verify that addresses have indeed been saved)
{ "AddressesFilename", "addresses.csv" },
{ "PublisherFilename", "publishers.txt" },
{ "PublishersFilename", "publishers.txt" },
// TODO(unassigned): replace with Monero's b32 publisher service
{ "PublisherURI", "https://downloads.getmonero.org/kovri/hosts.txt" },
// Below is only used for testing in-net download (this is *not* our default subscription)
//{ "PublisherURI", "http://udhdrtrcetjm5sxzskjyr5ztpeszydbh4dpl3pl4utgqqw2v4jna.b32.i2p/hosts.txt" },
};

/// @class AddressBookStorage
@@ -133,9 +136,11 @@ class AddressBookFilesystemStorage : public AddressBookStorage {
const std::map<std::string, i2p::data::IdentHash>& addresses);

private:
/// @brief Gets addresses filename (file list of saved addresses)
// TODO(anonimal): move Get* defaults here, refactor filename versus full path

/// @brief Gets addresses file (file list of saved addresses)
/// @return Const reference to addresses filename
const std::string& GetAddressesFilename() {
const std::string GetAddressesFilenamePath() {
return (GetAddressBookPath() / AddressBookDefaults.at("AddressesFilename")).string();
}

4 changes: 2 additions & 2 deletions src/client/i2p_tunnel/http_proxy.cc
Original file line number Diff line number Diff line change
@@ -309,8 +309,8 @@ void HTTPProxyHandler::HandleJumpService() {
}
auto base64 = m_Path.substr(pos + m_JumpService.at(0).size());
// We must decode
i2p::util::http::URI uri;
base64 = uri.Decode(base64);
i2p::util::http::HTTP uri;
base64 = uri.HTTPProxyDecode(base64);
// Insert into address book
LogPrint(eLogDebug,
"HTTPProxyHandler: jump service for ", m_Address,
2 changes: 1 addition & 1 deletion src/client/i2p_tunnel/i2p_tunnel.h
Original file line number Diff line number Diff line change
@@ -42,7 +42,7 @@
#include <string>

#include "identity.h"
#include "streaming.h"
#include "client/streaming.h"
#include "client/destination.h"
#include "client/i2p_service.h"

8 changes: 4 additions & 4 deletions src/core/reseed.cc
Original file line number Diff line number Diff line change
@@ -187,11 +187,11 @@ bool Reseed::FetchStream(
const std::string& url) {
LogPrint(eLogInfo, "Reseed: fetching stream from ", url);
// TODO(unassigned): abstract our downloading mechanism (see #168)
i2p::util::http::HTTP http;
if (!http.Download(url))
return false;
i2p::util::http::HTTP http(url);
if (!http.Download())
return false; // Download failed
// Replace our stream with downloaded stream
m_Stream = http.m_Stream;
m_Stream.assign(http.GetDownloadedContents());
// TODO(unassigned): replace with constants if this isn't rewritten by #155/#168
return ((m_Stream.size() > 0) && (m_Stream.size() <= 128 * 1024)); // Arbitrary size in bytes
}
361 changes: 267 additions & 94 deletions src/core/util/http.cc

Large diffs are not rendered by default.

254 changes: 194 additions & 60 deletions src/core/util/http.h
Original file line number Diff line number Diff line change
@@ -33,17 +33,21 @@
#ifndef SRC_CORE_UTIL_HTTP_H_
#define SRC_CORE_UTIL_HTTP_H_

// TODO(anonimal): cleanup headers

#include <boost/algorithm/string.hpp>
#include <boost/asio.hpp>
#include <boost/asio/ssl.hpp>
#include <boost/filesystem.hpp>
#include <boost/foreach.hpp>
#include <boost/lexical_cast.hpp>

#include <openssl/bn.h>
#include <openssl/err.h>
#include <openssl/ssl.h>
// cpp-netlib
#define BOOST_NETWORK_ENABLE_HTTPS
#include <boost/network/include/http/client.hpp>
#include <boost/network/uri.hpp>

#include <cstdint>
#include <fstream>
#include <iostream>
#include <map>
@@ -54,78 +58,208 @@
#include "log.h"
#include "reseed.h"

// TODO(anonimal): fix certificates for i2*.manas.ca and downloads.getmonero.org

namespace i2p {
namespace util {
namespace http {
namespace http { // TODO(anonimal): consider removing this namespace (its not needed)

/// @enum Timeout
/// @brief Constants used for HTTP timeout lengths when downloading
/// @notes Scoped to prevent namespace pollution (otherwise, purely stylistic)
enum struct Timeout : std::uint8_t {
// Seconds
Request = 45, // Java I2P defined
Receive = 30,
};

/// @class HTTPStorage
/// @brief Storage for class HTTP
class HTTPStorage {
public:
/// @brief Set URI path to test against future downloads
/// @param path URI path
/// @notes Needed in conjunction with ETag
void SetPath(
const std::string& path) {
m_Path.assign(path);
}

/// @brief Get previously set URI path
/// @notes Needed in conjunction with ETag
const std::string GetPreviousPath() {
return m_Path;
}

/// @brief Set ETag member from response header
void SetETag(
const std::string& etag) {
m_ETag.assign(etag);
}

/// @brief Get previously set ETag member from response header
/// @return ETag value
const std::string& GetPreviousETag() {
return m_ETag;
}

/// @brief Set Last-Modified member from response header
void SetLastModified(
const std::string& last_modified) {
m_LastModified.assign(last_modified);
}

// TODO(anonimal): refactor everything in this namespace with cpp-netlib
// Will require review/refactor of AddressBook
/// @brief Get previously set Last-Modified member from response header
/// @return Last-Modified value
const std::string& GetPreviousLastModified() {
return m_LastModified;
}

/// @class URI
/// @brief Provides functionality for preparing a URI
class URI {
/// @brief Sets downloaded contents to stream
/// @notes Called after completed download
void SetDownloadedContents(
const std::string& stream) {
m_Stream.assign(stream);
}

/// @brief Gets downloaded contents after successful download
/// @return String of downloaded contents
/// @notes Called after completed download
const std::string& GetDownloadedContents() {
return m_Stream;
}

private:
/// @var m_Path
/// @brief Path value from a 1st request that can be tested against later
/// @notes If path is same as previous request, apply required header values
std::string m_Path;

/// @var m_ETag
/// @brief ETag value from response header
/// @notes Used primarily for subscriptions. Can be extended to auto-update
std::string m_ETag;

/// @var m_LastModified
/// @brief Last-Modified value from response header
/// @notes Used primarily for subscriptions. Can be extended to auto-update
std::string m_LastModified;

/// @var m_Stream
/// @brief Downloaded contents
std::string m_Stream; // TODO(anonimal): consider refactoring into an actual stream
};

/// @class HTTP
/// @brief Provides functionality for implementing HTTP/S
/// @details URI is typically passed to ctor. Otherwise, see below.
/// @notes Vocabulary:
/// Clearnet: Connections made outside of the I2P network
/// In-net: Connections made within the I2P network
class HTTP : public HTTPStorage {
public:
/// @param uri The URI string to be parsed
/// @note The default port is 80, for HTTPS it is 443
/// @return False if URI is invalid, true if valid
bool Parse(
HTTP() {} // for HTTPProxy and tests
~HTTP() {}

HTTP(const std::string& uri)
: m_URI(uri) {}

/// @brief Set cpp-netlib URI object if not set with ctor
/// @param uri String URI (complete)
void SetURI(
const std::string& uri) {
// Remove existing URI if set
if (!m_URI.string().empty()) {
boost::network::uri::uri new_uri;
m_URI.swap(new_uri);
}
// Set new URI
m_URI.append(uri);
}

/// @brief Get initialized URI
/// @return cpp-netlib URI object
boost::network::uri::uri GetURI() {
return m_URI;
}

/// @brief Tests if TLD is I2P and set default ports for in-net downloading
/// @return True if TLD is .i2p
/// @notes The default port is 80, for HTTPS it is 443
/// @notes Removing this will require refactoring stream implementation
bool HostIsI2P();

/// @brief Downloads parameter URI
/// @param uri String URI
/// @details Sets member URI with param uri, calls Download()
/// @return Bool result of Download()
/// @notes Only used if URI not initialized with ctor
bool Download(
const std::string& uri);

/// @brief Download wrapper function for clearnet and in-net download
/// @return False on failure
bool Download();

/// @brief Decode URI in HTTP proxy request
/// @return The decoded URI as string
std::string Decode(
std::string HTTPProxyDecode(
const std::string& data);

// TODO(anonimal): consider Get/Set functions if we keep class URI
std::string m_Protocol, m_Host, m_Port, m_Path, m_Query;
};
private:
/// @brief Downloads over clearnet
/// @return False on failure
bool DownloadViaClearnet();

/// @brief Downloads within I2P
/// @return False on failure
/// @notes Used for address book and for future in-net autoupdates
bool DownloadViaI2P();

private:
/// @var m_URI
/// @brief cpp-netlib URI instance
boost::network::uri::uri m_URI;

// TODO(anonimal): consider removing typedefs after refactor
// TODO(anonimal): remove the following notes after refactor

/// @brief HTTP client object
/// @notes Currently only applies to clearnet download
typedef boost::network::http::client Client;

/// @brief HTTP client options object (timeout, SNI, etc.)
/// @notes Currently only applies to clearnet download
typedef boost::network::http::client::options Options;

/// @brief HTTP client request object (header, etc.)
/// @notes Currently only applies to clearnet download
typedef boost::network::http::client::request Request;

/// @brief HTTP client response object (body, status, etc.)
/// @notes Currently only applies to clearnet download
typedef boost::network::http::client::response Response;

/**
* Provides functionality for implementing HTTP
*/
class HTTP {
public:
/**
* @return the result of the download, or an empty string if it fails
*/
bool Download(
const std::string& address);

/**
* Header for HTTP requests.
* @return a string of the complete header
* @warning this function does NOT append an additional \r\n
*/
const std::string Header(
const std::string& path,
const std::string& host,
const std::string& version);

/**
* @return the content of the given HTTP stream without headers
*/
const std::string GetContent(
std::istream& response);

/**
* Merge chunks of an HTTP response.
*/
void MergeChunkedResponse(
// TODO(anonimal): remove after refactor
/// @brief Prepares header for in-net request
void PrepareI2PRequest();

// TODO(anonimal): remove after refactor
/// @brief Process in-net HTTP response
/// @return True if processing was successful
bool ProcessI2PResponse();

// TODO(anonimal): remove after refactor
/// @brief Merge chunks of an in-net HTTP response
void MergeI2PChunkedResponse(
std::istream& response,
std::ostream& merged);

public:
/**
* Used almost exclusively by Addressbook
*/
const std::string ETAG = "ETag";
const std::string IF_NONE_MATCH = "If-None-Match";
const std::string IF_MODIFIED_SINCE = "If-Modified-Since";
const std::string LAST_MODIFIED = "Last-Modified";
const std::string TRANSFER_ENCODING = "Transfer-Encoding";

public:
std::string m_Stream; // Downloaded stream
std::uint16_t m_Status; // HTTP Response Code
private:
// TODO(anonimal): remove after refactor
/// @brief In-net HTTP request and response
std::stringstream m_Request, m_Response;
};

} // namespace http
19 changes: 11 additions & 8 deletions src/tests/unit_tests/core/util/http.cc
Original file line number Diff line number Diff line change
@@ -34,26 +34,29 @@

#include "util/http.h"

#include <iostream>
#include <boost/network/uri.hpp>

BOOST_AUTO_TEST_SUITE(HTTPUtilityTests)

i2p::util::http::URI uri;
i2p::util::http::HTTP http;

BOOST_AUTO_TEST_CASE(UriParse) {
// Note: cpp-netlib has better tests.
// We simply test our implementation here.
BOOST_CHECK(uri.Parse("https://domain.org:8443/path/file.type"));
BOOST_CHECK(!uri.Parse("3;axc807uasdfh123m,nafsdklfj;;klj0a9u01q3"));
http.SetURI("https://domain.org:8443/path/file.type");
BOOST_CHECK(http.GetURI().is_valid());

http.SetURI("3;axc807uasdfh123m,nafsdklfj;;klj0a9u01q3");
BOOST_CHECK(!http.GetURI().is_valid());

http.SetURI("http://username:password@udhdrtrcetjm5sxzskjyr5ztpeszydbh4dpl3pl4utgqqw2v4jna.b32.i2p/hosts.txt");
BOOST_CHECK(http.GetURI().is_valid() && http.HostIsI2P());
}

BOOST_AUTO_TEST_CASE(DecodeEmptyUri) {
BOOST_CHECK_EQUAL(uri.Decode(""), "");
BOOST_CHECK_EQUAL(http.HTTPProxyDecode(""), "");
}

BOOST_AUTO_TEST_CASE(DecodeUri) {
BOOST_CHECK_EQUAL(uri.Decode("%20"), " ");
BOOST_CHECK_EQUAL(http.HTTPProxyDecode("%20"), " ");
}

BOOST_AUTO_TEST_SUITE_END()

0 comments on commit add429b

Please sign in to comment.