#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // Mark string for translation (alias for leatherman::locale::format) using leatherman::locale::_; namespace hocon { namespace config_parser { using namespace std; shared_value parse(shared_ptr document, shared_origin origin, config_parse_options options, shared_include_context include_context) { parse_context context {options.get_syntax(), origin, document, simple_includer::make_full(options.get_includer()), include_context}; return context.parse(); } parse_context::parse_context(config_syntax flavor, shared_origin origin, shared_ptr document, shared_ptr includer, shared_include_context include_context) : _line_number(1), _document(document), _includer(includer), _include_context(include_context), _flavor(flavor), _base_origin(origin), array_count(0) {} shared_origin parse_context::line_origin() const { return _base_origin->with_line_number(_line_number); } path parse_context::full_current_path() const { // pathStack has top of stack at front if (_path_stack.empty()) { throw bug_or_broken_exception(_("Bug in parser; tried to get current path when at root")); } else { return path {_path_stack.front()}; } } shared_value parse_context::parse_value(shared_node_value n, vector& comments) { auto starting_array_count = array_count; shared_value v = [&]() -> shared_value { if (auto val = dynamic_pointer_cast(n)) { return val->get_value(); } else if (auto val = dynamic_pointer_cast(n)) { return parse_object(val); } else if (auto val = dynamic_pointer_cast(n)) { return parse_array(val); } else if (auto val = dynamic_pointer_cast(n)) { return parse_concatenation(val); } else { auto &deref_n = *n; throw parse_exception(*line_origin(), _("Expecting a value but got wrong node type: {1}", typeid(deref_n).name())); } }(); if (!comments.empty()) { auto old_origin = dynamic_pointer_cast(v->origin()); if (!old_origin) { throw bug_or_broken_exception(_("origin should be a simple_config_origin")); } v = v->with_origin(old_origin->prepend_comments(move(comments))); } if (array_count != starting_array_count) { throw bug_or_broken_exception(_("Bug in config parser: unbalanced array count")); } return v; } shared_object parse_context::create_value_under_path(path p, shared_value value) { // for path foo.bar, we are creating { "foo" : { "bar" : value } } vector keys; shared_string key = p.first(); path remaining = p.remainder(); while (key != nullptr) { keys.push_back(key); if (remaining.empty()) { break; } else { key = remaining.first(); remaining = remaining.remainder(); } } auto current = keys.end(); current--; auto new_value = unordered_map({ {**current, value} }); shared_object obj = make_shared(value->origin()->with_comments(vector{}), new_value); while (current != keys.begin()) { current--; new_value = unordered_map({ {**current, obj} }); obj = make_shared(value->origin()->with_comments(vector{}), new_value); } return obj; } void parse_context::parse_include(unordered_map & values, shared_ptr n) { shared_object obj; switch (n->kind()) { case config_include_kind::FILE: obj = dynamic_pointer_cast(_includer->include_file(_include_context, n->name())); break; case config_include_kind::CLASSPATH: // TODO: implement include_resource (?) throw config_exception(_("full_includer::include_resource not implemented")); break; case config_include_kind::HEURISTIC: obj = dynamic_pointer_cast(_includer->include(_include_context, n->name())); break; default: throw config_exception(_("should not be reached")); break; } // we really should make this work, but for now throwing an // exception is better than producing an incorrect result. // See https://github.com/typesafehub/config/issues/160 if (array_count > 0 && obj->get_resolve_status() != resolve_status::RESOLVED) { throw config_exception(leatherman::locale::translate("Due to current limitations of the config parser, when an include statement is nested inside a list value,\n${} substitutions inside the included file cannot be resolved correctly. Either move the include outside of the list value or\nremove the ${} statements from the included file.")); } if (!_path_stack.empty()) { auto prefix = full_current_path(); obj = dynamic_pointer_cast(obj->relativized(prefix.to_string())); } for (auto &pair : *obj) { auto &key = pair.first; auto &v = pair.second; auto iter = values.find(key); if (iter != values.end()) { auto existing = values[key]; values[key] = dynamic_pointer_cast(v->with_fallback(existing)); } else { values.emplace(key, v); } } } shared_object parse_context::parse_object(shared_node_object n) { unordered_map values; auto object_origin = line_origin(); bool last_was_newline = false; auto nodes = n->children(); vector comments; for (size_t i = 0; i < nodes.size(); ++i) { auto node = nodes.at(i); if (auto comment = dynamic_pointer_cast(node)) { last_was_newline = false; comments.push_back(comment->comment_text()); } else if (auto singletoken = dynamic_pointer_cast(node)) { if (tokens::is_newline(singletoken->get_token())) { _line_number++; if (last_was_newline) { // Drop all comments if there was a blank line and start a new comment block comments.clear(); } last_was_newline = true; } } else if (auto include = dynamic_pointer_cast(node)) { if (_flavor != config_syntax::JSON) { parse_include(values, include); last_was_newline = false; } } else if (auto field = dynamic_pointer_cast(node)) { last_was_newline = false; auto path = field->path()->get_path(); auto field_comments = field->comments(); comments.insert(comments.end(), field_comments.begin(), field_comments.end()); // path must be on-stack while we parse the value _path_stack.push_back(path); if (field->separator() == tokens::plus_equals_token()) { // we really should make this work, but for now throwing // an exception is better than producing an incorrect // result. See // https://github.com/typesafehub/config/issues/160 if (array_count > 0) { throw parse_exception(*line_origin(), leatherman::locale::translate("Due to current limitations of the config parser, += does not work nested inside a list. += expands to a ${} substitution and the path in ${} cannot currently refer to list elements. You might be able to move the += outside of the list and then refer to it from inside the list with ${}.")); } // because we will put it in an array after the fact so // we want this to be incremented during the parseValue // below in order to throw the above exception. array_count += 1; } auto value_node = field->get_value(); // comments from the key token go to the value token auto new_value = parse_value(value_node, comments); if (field->separator() == tokens::plus_equals_token()) { array_count -= 1; vector concat; concat.reserve(2); auto previous_ref = make_shared(new_value->origin(), make_shared(full_current_path(), true)); auto list = make_shared(new_value->origin(), vector({new_value})); concat.push_back(previous_ref); concat.push_back(list); new_value = config_concatenation::concatenate(concat); } // Grab any trailing comments on the same line if (i < nodes.size() - 1) { ++i; while (i < nodes.size()) { if (auto comment = dynamic_pointer_cast(nodes.at(i))) { auto old_origin = dynamic_pointer_cast(new_value->origin()); if (!old_origin) { throw bug_or_broken_exception(_("expected origin to be simple_config_origin")); } new_value = new_value->with_origin(old_origin->append_comments({comment->comment_text()})); break; } else if (auto curr = dynamic_pointer_cast(nodes.at(i))) { if (curr->get_token() == tokens::comma_token() || tokens::is_ignored_whitespace(curr->get_token())) { // keep searching, as there could still be a comment } else { --i; break; } } else { --i; break; } ++i; } } _path_stack.pop_back(); auto key = path.first(); auto remaining = path.remainder(); if (remaining.empty()) { auto existing = values.find(*key); if (existing != values.end()) { // In strict JSON, dups should be an error; while in // our custom config language, they should be merged // if the value is an object (or substitution that // could become an object). if (_flavor == config_syntax::JSON) { throw parse_exception(*line_origin(), _("JSON does not allow duplicate fields: '{1}' was already seen at {2}", *key, existing->second->origin()->description())); } else { new_value = dynamic_pointer_cast(new_value->with_fallback(existing->second)); assert(new_value); } } values[*key] = new_value; } else { if (_flavor == config_syntax::JSON) { throw new bug_or_broken_exception(_("somehow got multi-element path in JSON mode")); } shared_object obj = create_value_under_path(remaining, new_value); auto existing = values.find(*key); if (existing != values.end()) { obj = dynamic_pointer_cast(obj->with_fallback(existing->second)); assert(obj); } values[*key] = obj; } } } return make_shared(object_origin, move(values)); } static shared_ptr as_origin(shared_origin o) { auto simple_o = dynamic_pointer_cast(o); if (!simple_o) { throw bug_or_broken_exception(_("origin was not a simple_config_origin")); } return simple_o; } shared_value parse_context::parse_array(shared_node_array n) { ++array_count; auto array_origin = line_origin(); auto values = vector(); bool last_was_new_line = false; auto comments = vector(); shared_value v; for (auto node : n->children()) { if (auto comment = dynamic_pointer_cast(node)) { comments.push_back(comment->comment_text()); last_was_new_line = false; } else if (auto singletoken = dynamic_pointer_cast(node)) { if (tokens::is_newline(singletoken->get_token())) { _line_number++; if (last_was_new_line && v == nullptr) { comments.clear(); } else if (v) { values.push_back(v->with_origin(as_origin(v->origin())->append_comments(move(comments)))); comments.clear(); v = nullptr; } last_was_new_line = true; } } else if (auto value = dynamic_pointer_cast(node)) { last_was_new_line = false; if (v) { values.push_back(v->with_origin(as_origin(v->origin())->append_comments(move(comments)))); comments.clear(); } v = parse_value(value, comments); } } // There shouldn't be any comments at this point, but add them just in case if (v) { values.push_back(v->with_origin(as_origin(v->origin())->append_comments(move(comments)))); } --array_count; return make_shared(move(array_origin), move(values)); } shared_value parse_context::parse_concatenation(shared_node_concatenation n) { if (_flavor == config_syntax::JSON) { throw bug_or_broken_exception(_("Found a concatenation node in JSON")); } vector values; for (auto& node : n->children()) { if (auto value_node = dynamic_pointer_cast(node)) { vector comments; values.push_back(parse_value(value_node, comments)); } } return config_concatenation::concatenate(move(values)); } shared_value parse_context::parse() { shared_value result; vector comments; bool last_was_newline = false; for (auto&& node : _document->children()) { if (auto ptr = dynamic_pointer_cast(node)) { comments.push_back(ptr->comment_text()); last_was_newline = false; } else if (auto ptr = dynamic_pointer_cast(node)) { auto t = ptr->get_token(); if (t->get_token_type() == token_type::NEWLINE) { ++_line_number; if (last_was_newline && !result) { comments.clear(); } else if (result) { auto origin = dynamic_pointer_cast(result->origin()); assert(origin); result = result->with_origin(origin->append_comments(move(comments))); break; } last_was_newline = true; } } else if (auto ptr = dynamic_pointer_cast(node)) { result = parse_value(ptr, comments); last_was_newline = false; } } return result; } }} // namespace hocon::config_parser