33 #include "hazelcast/util/Preconditions.h"
34 #include "hazelcast/client/aws/aws_client.h"
35 #include "hazelcast/client/client_properties.h"
36 #include "hazelcast/client/config/client_aws_config.h"
37 #include "hazelcast/logger.h"
39 #ifdef HZ_BUILD_WITH_SSL
43 #include <boost/algorithm/string/replace.hpp>
44 #include <boost/date_time.hpp>
45 #include <boost/property_tree/xml_parser.hpp>
46 #include <boost/property_tree/json_parser.hpp>
48 #include "hazelcast/client/aws/utility/aws_url_encoder.h"
49 #include "hazelcast/client/aws/impl/Constants.h"
50 #include "hazelcast/client/aws/security/ec2_request_signer.h"
51 #include "hazelcast/client/aws/impl/Filter.h"
52 #include "hazelcast/client/aws/impl/DescribeInstances.h"
53 #include "hazelcast/client/aws/utility/cloud_utility.h"
54 #include "hazelcast/util/SyncHttpsClient.h"
55 #include "hazelcast/util/SyncHttpClient.h"
58 #include <openssl/ssl.h>
64 std::string ec2_request_signer::NEW_LINE =
"\n";
65 size_t ec2_request_signer::DATE_LENGTH = 8;
67 ec2_request_signer::ec2_request_signer(
const config::client_aws_config &aws_config,
68 const std::string ×tamp,
69 const std::string &endpoint) : aws_config_(aws_config),
70 timestamp_(timestamp),
74 ec2_request_signer::~ec2_request_signer() =
default;
76 std::string ec2_request_signer::sign(
const std::unordered_map<std::string, std::string> &attributes) {
77 std::string canonicalRequest = get_canonicalized_request(attributes);
78 std::string stringToSign = create_string_to_sign(canonicalRequest);
79 std::vector<unsigned char> signingKey = derive_signing_key();
81 return create_signature(stringToSign, signingKey);
84 std::string ec2_request_signer::create_formatted_credential()
const {
85 std::stringstream out;
86 out << aws_config_.get_access_key() <<
'/' << timestamp_.substr(0, DATE_LENGTH) <<
'/'
87 << aws_config_.get_region() <<
'/' <<
"ec2/aws4_request";
91 std::string ec2_request_signer::get_canonicalized_query_string(
92 const std::unordered_map<std::string, std::string> &attributes)
const {
93 std::vector<std::string> components = get_list_of_entries(attributes);
94 std::sort(components.begin(), components.end());
95 return get_canonicalized_query_string(components);
99 std::string ec2_request_signer::get_canonicalized_request(
100 const std::unordered_map<std::string, std::string> &attributes)
const {
101 std::ostringstream out;
102 out << impl::Constants::GET << NEW_LINE
104 << get_canonicalized_query_string(attributes) << NEW_LINE
105 << get_canonical_headers() << NEW_LINE
106 <<
"host" << NEW_LINE
107 << sha256_hashhex(
"");
111 std::string ec2_request_signer::get_canonical_headers()
const {
112 std::ostringstream out;
113 out <<
"host:" << endpoint_ << NEW_LINE;
117 std::string ec2_request_signer::get_canonicalized_query_string(
const std::vector<std::string> &list)
const {
118 std::ostringstream result;
119 std::vector<std::string>::const_iterator it = list.begin();
122 for (; it != list.end(); ++it) {
123 result <<
"&" << *it;
128 std::vector<std::string> ec2_request_signer::get_list_of_entries(
129 const std::unordered_map<std::string, std::string> &entries)
const {
130 std::vector<std::string> components;
131 for (
const auto &entry: entries) {
132 components.push_back(format_attribute(entry.first, entry.second));
137 std::string ec2_request_signer::format_attribute(
const std::string &key,
const std::string &value) {
138 std::ostringstream out;
139 out << utility::aws_url_encoder::url_encode(key) <<
'='
140 << utility::aws_url_encoder::url_encode(value);
145 std::string ec2_request_signer::create_string_to_sign(
const std::string &canonical_request)
const {
146 std::ostringstream out;
147 out << impl::Constants::SIGNATURE_METHOD_V4 << NEW_LINE
148 << timestamp_ << NEW_LINE
149 << get_credential_scope() << NEW_LINE
150 << sha256_hashhex(canonical_request);
154 std::string ec2_request_signer::get_credential_scope()
const {
157 std::ostringstream out;
158 out << timestamp_.substr(0, DATE_LENGTH) <<
"/" << aws_config_.get_region() <<
"/ec2/aws4_request";
163 std::vector<unsigned char> ec2_request_signer::derive_signing_key()
const {
164 const std::string &signKey = aws_config_.get_secret_key();
165 std::string dateStamp = timestamp_.substr(0, DATE_LENGTH);
169 unsigned char kDate[32];
170 std::string key = std::string(
"AWS4") + signKey;
171 int kDateLen = hmac_sh_a256_bytes(key, dateStamp, kDate);
173 unsigned char kRegion[32];
174 int kRegionLen = hmac_sh_a256_bytes(kDate, kDateLen, aws_config_.get_region(), kRegion);
176 unsigned char kService[32];
177 int kServiceLen = hmac_sh_a256_bytes(kRegion, kRegionLen,
"ec2", kService);
179 std::vector<unsigned char> mSigning(32);
180 hmac_sh_a256_bytes(kService, kServiceLen,
"aws4_request", &mSigning[0]);
185 std::string ec2_request_signer::create_signature(
const std::string &string_to_sign,
186 const std::vector<unsigned char> &signing_key)
const {
187 return hmac_sh_a256_hex(signing_key, string_to_sign);
190 std::string ec2_request_signer::hmac_sh_a256_hex(
const std::vector<unsigned char> &key,
191 const std::string &msg)
const {
192 unsigned char hash[32];
194 unsigned int len = hmac_sh_a256_bytes(key, msg, hash);
196 return convert_to_hex_string(hash, len);
199 std::string ec2_request_signer::convert_to_hex_string(
const unsigned char *buffer,
unsigned int len)
const {
200 std::stringstream ss;
201 ss << std::hex << std::setfill(
'0');
202 for (
unsigned int i = 0; i < len; i++) {
203 ss << std::hex << std::setw(2) << (
unsigned int) buffer[i];
209 unsigned int ec2_request_signer::hmac_sh_a256_bytes(
const void *key,
int key_len,
const std::string &msg,
210 unsigned char *hash)
const {
211 return hmac_sh_a256_bytes(key, key_len, (
unsigned char *) &msg[0], msg.length(),
215 unsigned int ec2_request_signer::hmac_sh_a256_bytes(
const std::string &key,
const std::string &msg,
216 unsigned char *hash)
const {
217 return hmac_sh_a256_bytes(&key[0], (
int) key.length(), (
unsigned char *) &msg[0], msg.length(),
221 unsigned int ec2_request_signer::hmac_sh_a256_bytes(
const std::vector<unsigned char> &key,
222 const std::string &msg,
223 unsigned char *hash)
const {
224 return hmac_sh_a256_bytes(&key[0], (
int) key.size(), (
unsigned char *) &msg[0], msg.length(),
228 unsigned int ec2_request_signer::hmac_sh_a256_bytes(
const void *key_buffer,
int key_len,
229 const unsigned char *data,
231 unsigned char *hash)
const {
232 #if OPENSSL_VERSION_NUMBER > 0x10100000L
233 HMAC_CTX *hmac = HMAC_CTX_new();
235 HMAC_CTX *hmac =
new HMAC_CTX;
239 HMAC_Init_ex(hmac, key_buffer, key_len, EVP_sha256(), NULL);
240 HMAC_Update(hmac, data, data_len);
241 unsigned int len = 32;
242 HMAC_Final(hmac, hash, &len);
244 #if OPENSSL_VERSION_NUMBER > 0x10100000L
247 HMAC_CTX_cleanup(hmac);
254 std::string ec2_request_signer::sha256_hashhex(
const std::string &in)
const {
256 unsigned int hashLen = 0;
257 unsigned char hash[EVP_MAX_MD_SIZE];
259 EVP_MD_CTX_init(&ctx);
260 EVP_DigestInit_ex(&ctx, EVP_sha256(), NULL);
261 EVP_DigestUpdate(&ctx, in.c_str(), in.size());
262 EVP_DigestFinal_ex(&ctx, hash, &hashLen);
263 EVP_MD_CTX_cleanup(&ctx);
264 return convert_to_hex_string(hash, hashLen);
266 unsigned char hash[SHA256_DIGEST_LENGTH];
268 SHA256_Init(&sha256);
269 SHA256_Update(&sha256, in.c_str(), in.size());
270 SHA256_Final(hash, &sha256);
272 return convert_to_hex_string(hash, SHA256_DIGEST_LENGTH);
278 const char *Constants::DATE_FORMAT =
"%Y%m%dT%H%M%SZ";
279 const char *Constants::DOC_VERSION =
"2016-11-15";
280 const char *Constants::SIGNATURE_METHOD_V4 =
"AWS4-HMAC-SHA256";
281 const char *Constants::GET =
"GET";
282 const char *Constants::ECS_CREDENTIALS_ENV_VAR_NAME =
"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI";
284 Filter::Filter() =
default;
294 void Filter::add_filter(
const std::string &name,
const std::string &value) {
295 std::stringstream out;
296 unsigned long index = filters_.size() + 1;
297 out <<
"Filter." << index <<
".Name";
298 filters_[out.str()] = name;
301 out <<
"Filter." << index <<
".Value.1";
302 filters_[out.str()] = value;
305 const std::unordered_map<std::string, std::string> &Filter::get_filters() {
309 const std::string DescribeInstances::QUERY_PREFIX =
"/?";
310 const std::string DescribeInstances::IAM_ROLE_ENDPOINT =
"169.254.169.254";
311 const std::string DescribeInstances::IAM_ROLE_QUERY =
"/latest/meta-data/iam/security-credentials/";
312 const std::string DescribeInstances::IAM_TASK_ROLE_ENDPOINT =
"169.254.170.2";
314 DescribeInstances::DescribeInstances(std::chrono::steady_clock::duration timeout,
315 config::client_aws_config &aws_config,
const std::string &endpoint,
316 logger &lg) : timeout_(timeout), aws_config_(aws_config),
317 endpoint_(endpoint), logger_(lg) {
318 check_keys_from_iam_roles();
320 std::string timeStamp = get_formatted_timestamp();
321 rs_ = std::unique_ptr<security::ec2_request_signer>(
322 new security::ec2_request_signer(aws_config, timeStamp, endpoint));
323 attributes_[
"Action"] =
"DescribeInstances";
324 attributes_[
"Version"] = impl::Constants::DOC_VERSION;
325 attributes_[
"X-Amz-Algorithm"] = impl::Constants::SIGNATURE_METHOD_V4;
326 attributes_[
"X-Amz-Credential"] = rs_->create_formatted_credential();
327 attributes_[
"X-Amz-Date"] = timeStamp;
328 attributes_[
"X-Amz-SignedHeaders"] =
"host";
329 attributes_[
"X-Amz-Expires"] =
"30";
333 DescribeInstances::~DescribeInstances() =
default;
335 std::unordered_map<std::string, std::string> DescribeInstances::execute() {
336 std::string signature = rs_->sign(attributes_);
337 attributes_[
"X-Amz-Signature"] = signature;
339 std::istream &stream = call_service();
340 return utility::cloud_utility::unmarshal_the_response(stream, logger_);
343 std::string DescribeInstances::get_formatted_timestamp() {
344 using namespace boost::posix_time;
345 ptime now = second_clock::universal_time();
347 std::ostringstream out;
348 std::locale timeLocale(out.getloc(),
new time_facet(impl::Constants::DATE_FORMAT));
349 out.imbue(timeLocale);
354 std::istream &DescribeInstances::call_service() {
355 std::string query = rs_->get_canonicalized_query_string(attributes_);
356 https_client_ = std::unique_ptr<util::SyncHttpsClient>(
357 new util::SyncHttpsClient(endpoint_.c_str(), QUERY_PREFIX + query, timeout_));
358 return https_client_->connect_and_get_response();
361 void DescribeInstances::check_keys_from_iam_roles() {
362 if (aws_config_.get_access_key().empty() || !aws_config_.get_iam_role().empty()) {
363 try_get_default_iam_role();
364 if (!aws_config_.get_iam_role().empty()) {
365 get_keys_from_iam_role();
367 get_keys_from_iam_task_role();
372 void DescribeInstances::try_get_default_iam_role() {
374 if (!(aws_config_.get_iam_role().empty() || aws_config_.get_iam_role() ==
"DEFAULT")) {
379 util::SyncHttpClient httpClient(IAM_ROLE_ENDPOINT, IAM_ROLE_QUERY);
380 std::string roleName;
381 std::istream &responseStream = httpClient.open_connection();
382 responseStream >> roleName;
383 aws_config_.set_iam_role(roleName);
384 }
catch (exception::io &e) {
385 BOOST_THROW_EXCEPTION(exception::invalid_configuration(
"tryGetDefaultIamRole",
387 "Invalid Aws Configuration. ") +
392 void DescribeInstances::get_keys_from_iam_task_role() {
395 #if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
396 #pragma warning(push)
397 #pragma warning(disable: 4996)
399 const char *uri = std::getenv(Constants::ECS_CREDENTIALS_ENV_VAR_NAME);
400 #if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
404 BOOST_THROW_EXCEPTION(exception::illegal_argument(
"getKeysFromIamTaskRole",
405 "Could not acquire credentials! Did not find declared AWS access key or IAM Role, and could not discover IAM Task Role or default role."));
408 util::SyncHttpClient httpClient(IAM_TASK_ROLE_ENDPOINT, uri);
411 std::istream &istream = httpClient.open_connection();
412 parse_and_store_role_creds(istream);
413 }
catch (exception::iexception &e) {
414 std::stringstream out;
415 out <<
"Unable to retrieve credentials from IAM Task Role. URI: " << uri <<
". \n " << e.what();
416 BOOST_THROW_EXCEPTION(
417 exception::invalid_configuration(
"getKeysFromIamTaskRole", out.str()));
421 void DescribeInstances::get_keys_from_iam_role() {
422 std::string query =
"/latest/meta-data/iam/security-credentials/" + aws_config_.get_iam_role();
424 util::SyncHttpClient httpClient(IAM_ROLE_ENDPOINT, query);
427 std::istream &istream = httpClient.open_connection();
428 parse_and_store_role_creds(istream);
429 }
catch (exception::iexception &e) {
430 std::stringstream out;
431 out <<
"Unable to retrieve credentials from IAM Task Role. URI: " << query <<
". \n "
433 BOOST_THROW_EXCEPTION(
434 exception::invalid_configuration(
"getKeysFromIamRole", out.str()));
438 void DescribeInstances::parse_and_store_role_creds(std::istream &in) {
439 utility::cloud_utility::unmarshal_json_response(in, aws_config_, attributes_);
445 void DescribeInstances::add_filters() {
447 if (!aws_config_.get_tag_key().empty()) {
448 if (!aws_config_.get_tag_value().empty()) {
449 filter.add_filter(std::string(
"tag:") + aws_config_.get_tag_key(), aws_config_.get_tag_value());
451 filter.add_filter(
"tag-key", aws_config_.get_tag_key());
453 }
else if (!aws_config_.get_tag_value().empty()) {
454 filter.add_filter(
"tag-value", aws_config_.get_tag_value());
457 if (!aws_config_.get_security_group_name().empty()) {
458 filter.add_filter(
"instance.group-name", aws_config_.get_security_group_name());
461 filter.add_filter(
"instance-state-name",
"running");
462 const std::unordered_map<std::string, std::string> &filters = filter.get_filters();
463 attributes_.insert(filters.begin(), filters.end());
469 std::string aws_url_encoder::url_encode(
const std::string &value) {
470 std::string result = escape_encode(value);
471 boost::replace_all(result,
"+",
"%20");
475 std::string aws_url_encoder::escape_encode(
const std::string &value) {
476 std::ostringstream escaped;
480 for (std::string::const_iterator i = value.begin(), n = value.end(); i != n; ++i) {
481 std::string::value_type c = (*i);
484 if (isalnum(c) || c ==
'-' || c ==
'_' || c ==
'.' || c ==
'~') {
490 escaped << std::uppercase;
491 escaped <<
'%' << std::setw(2) << int((
unsigned char) c);
492 escaped << std::nouppercase;
495 return escaped.str();
498 std::unordered_map<std::string, std::string> cloud_utility::unmarshal_the_response(std::istream &stream,
500 std::unordered_map<std::string, std::string> privatePublicPairs;
504 pt::read_xml(stream, tree);
505 }
catch (pt::xml_parser_error &e) {
507 boost::str(boost::format(
"The parsed xml stream has errors: %1%") % e.what()));
508 return privatePublicPairs;
513 for (pt::ptree::value_type &item : tree.get_child(
"DescribeInstancesResponse.reservationSet")) {
514 for (pt::ptree::value_type &instanceItem : item.second.get_child(
"instancesSet")) {
515 auto privateIp = instanceItem.second.get_optional<std::string>(
"privateIpAddress");
516 auto publicIp = instanceItem.second.get_optional<std::string>(
"ipAddress");
518 auto prIp = privateIp.value_or(
"");
519 auto pubIp = publicIp.value_or(
"");
522 privatePublicPairs[prIp] = pubIp;
524 boost::str(boost::format(
"Accepting EC2 instance [%1%][%2%]")
525 % instanceItem.second.get_optional<std::string>(
"tagset.item.value").value_or(
"")
531 return privatePublicPairs;
534 void cloud_utility::unmarshal_json_response(std::istream &stream, config::client_aws_config &aws_config,
535 std::unordered_map<std::string, std::string> &attributes) {
537 pt::read_json(stream, json);
538 aws_config.set_access_key(json.get_optional<std::string>(
"AccessKeyId").get_value_or(
""));
539 aws_config.set_secret_key(json.get_optional<std::string>(
"SecretAccessKey").get_value_or(
""));
540 attributes[
"X-Amz-Security-Token"] = json.get_optional<std::string>(
"Token").get_value_or(
"");
545 aws_client::aws_client(std::chrono::steady_clock::duration timeout, config::client_aws_config &aws_config,
546 const client_properties &client_properties, logger &lg) : timeout_(timeout),
547 aws_config_(aws_config), logger_(lg) {
548 this->endpoint_ = aws_config.get_host_header();
549 if (!aws_config.get_region().empty() && aws_config.get_region().length() > 0) {
550 if (aws_config.get_host_header().find(
"ec2.") != 0) {
551 BOOST_THROW_EXCEPTION(exception::invalid_configuration(
"aws_client::aws_client",
552 "HostHeader should start with \"ec2.\" prefix"));
554 boost::replace_all(this->endpoint_,
"ec2.", std::string(
"ec2.") + aws_config.get_region() +
".");
557 aws_member_port_ = client_properties.get_integer(client_properties.get_aws_member_port());
558 if (aws_member_port_ < 0 || aws_member_port_ > 65535) {
559 BOOST_THROW_EXCEPTION(
560 exception::invalid_configuration(
"aws_client::aws_client",
561 (boost::format(
"Configured aws member port %1% is not "
562 "a valid port number. It should be between 0-65535 inclusive.")
563 % aws_member_port_).str()));
567 std::unordered_map<address, address> aws_client::get_addresses() {
568 auto addr_pair_map = impl::DescribeInstances(timeout_,aws_config_, endpoint_, logger_).execute();
569 std::unordered_map<address, address> addr_map;
570 addr_map.reserve(addr_pair_map.size());
571 for (
const auto &addr_pair : addr_pair_map) {
572 addr_map.emplace(address{addr_pair.first, aws_member_port_},
573 address{addr_pair.second, aws_member_port_});
581 namespace hazelcast {
584 aws_client::aws_client(std::chrono::steady_clock::duration timeout, config::client_aws_config &aws_config,
585 const client_properties &client_properties, logger &lg) {
586 util::Preconditions::check_ssl(
"aws_client::aws_client");
589 std::unordered_map<address, address> aws_client::get_addresses() {
590 util::Preconditions::check_ssl(
"aws_client::get_addresses");