"
# doc = Nokogiri.HTML5(html)
# # raises ArgumentError: Attributes per element limit exceeded
# doc = Nokogiri.HTML5(html, max_attributes: -1)
#
# == HTML Serialization
#
# After parsing HTML, it may be serialized using any of the {Nokogiri::XML::Node} serialization
# methods. In particular, {XML::Node#serialize}, {XML::Node#to_html}, and {XML::Node#to_s} will
# serialize a given node and its children. (This is the equivalent of JavaScript's
# +Element.outerHTML+.) Similarly, {XML::Node#inner_html} will serialize the children of a given
# node. (This is the equivalent of JavaScript's +Element.innerHTML+.)
#
# doc = Nokogiri::HTML5("
Hello world!")
# puts doc.serialize
# # =>
Hello world!
#
# Due to quirks in how HTML is parsed and serialized, it's possible for a DOM tree to be
# serialized and then re-parsed, resulting in a different DOM. Mostly, this happens with DOMs
# produced from invalid HTML. Unfortunately, even valid HTML may not survive serialization and
# re-parsing.
#
# In particular, a newline at the start of +pre+, +listing+, and +textarea+ elements is ignored by
# the parser.
#
# doc = Nokogiri::HTML5(<<-EOF)
#
#
# Content
# EOF
# puts doc.at('/html/body/pre').serialize
# # =>
Content
#
# In this case, the original HTML is semantically equivalent to the serialized version. If the
# +pre+, +listing+, or +textarea+ content starts with two newlines, the first newline will be
# stripped on the first parse and the second newline will be stripped on the second, leading to
# semantically different DOMs. Passing the parameter
preserve_newline: true will cause
# two or more newlines to be preserved. (A single leading newline will still be removed.)
#
# doc = Nokogiri::HTML5(<<-EOF)
#
#
#
# Content
# EOF
# puts doc.at('/html/body/listing').serialize(preserve_newline: true)
# # =>
# #
# # Content
#
# == Encodings
#
# Nokogiri always parses HTML5 using {https://en.wikipedia.org/wiki/UTF-8 UTF-8}; however, the
# encoding of the input can be explicitly selected via the optional +encoding+ parameter. This is
# most useful when the input comes not from a string but from an IO object.
#
# When serializing a document or node, the encoding of the output string can be specified via the
# +:encoding+ options. Characters that cannot be encoded in the selected encoding will be encoded
# as {https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references HTML numeric
# entities}.
#
# frag = Nokogiri::HTML5.fragment('
아는 길도 물어가라')
# html = frag.serialize(encoding: 'US-ASCII')
# puts html
# # =>
아는 길도 물어가라
# frag = Nokogiri::HTML5.fragment(html)
# puts frag.serialize
# # =>
아는 길도 물어가라
#
# (There's a {https://bugs.ruby-lang.org/issues/15033 bug} in all current versions of Ruby that
# can cause the entity encoding to fail. Of the mandated supported encodings for HTML, the only
# encoding I'm aware of that has this bug is
'ISO-2022-JP'. We recommend avoiding this
# encoding.)
#
# == Notes
#
# * The {Nokogiri::HTML5.fragment} function takes a string and parses it
# as a HTML5 document. The ++, ++, and ++ elements are
# removed from this document, and any children of these elements that remain
# are returned as a {Nokogiri::HTML5::DocumentFragment}.
#
# * The {Nokogiri::HTML5.parse} function takes a string and passes it to the
#
gumbo_parse_with_options method, using the default options.
# The resulting Gumbo parse tree is then walked.
#
# * Instead of uppercase element names, lowercase element names are produced.
#
# * Instead of returning +unknown+ as the element name for unknown tags, the
# original tag name is returned verbatim.
#
# Since v1.12.0
module HTML5
class << self
# Parse an HTML 5 document. Convenience method for {Nokogiri::HTML5::Document.parse}
def parse(string, url = nil, encoding = nil, **options, &block)
Document.parse(string, url, encoding, **options, &block)
end
# Parse a fragment from +string+. Convenience method for
# {Nokogiri::HTML5::DocumentFragment.parse}.
def fragment(string, encoding = nil, **options)
DocumentFragment.parse(string, encoding, options)
end
# :nodoc:
def read_and_encode(string, encoding)
# Read the string with the given encoding.
if string.respond_to?(:read)
string = if encoding.nil?
string.read
else
string.read(encoding: encoding)
end
else
# Otherwise the string has the given encoding.
string = string.to_s
if encoding
string = string.dup
string.force_encoding(encoding)
end
end
# convert to UTF-8
if string.encoding != Encoding::UTF_8
string = reencode(string)
end
string
end
private
# Charset sniffing is a complex and controversial topic that understandably isn't done _by
# default_ by the Ruby Net::HTTP library. This being said, it is a very real problem for
# consumers of HTML as the default for HTML is iso-8859-1, most "good" producers use utf-8, and
# the Gumbo parser *only* supports utf-8.
#
# Accordingly, Nokogiri::HTML4::Document.parse provides limited encoding detection. Following
# this lead, Nokogiri::HTML5 attempts to do likewise, while attempting to more closely follow
# the HTML5 standard.
#
# http://bugs.ruby-lang.org/issues/2567
# http://www.w3.org/TR/html5/syntax.html#determining-the-character-encoding
#
def reencode(body, content_type = nil)
if body.encoding == Encoding::ASCII_8BIT
encoding = nil
# look for a Byte Order Mark (BOM)
initial_bytes = body[0..2].bytes
if initial_bytes[0..2] == [0xEF, 0xBB, 0xBF]
encoding = Encoding::UTF_8
elsif initial_bytes[0..1] == [0xFE, 0xFF]
encoding = Encoding::UTF_16BE
elsif initial_bytes[0..1] == [0xFF, 0xFE]
encoding = Encoding::UTF_16LE
end
# look for a charset in a content-encoding header
if content_type
encoding ||= content_type[/charset=["']?(.*?)($|["';\s])/i, 1]
end
# look for a charset in a meta tag in the first 1024 bytes
unless encoding
data = body[0..1023].gsub(/|\Z)/m, "")
data.scan(/
/im).each do |meta|
encoding ||= meta[/charset=["']?([^>]*?)($|["'\s>])/im, 1]
end
end
# if all else fails, default to the official default encoding for HTML
encoding ||= Encoding::ISO_8859_1
# change the encoding to match the detected or inferred encoding
body = body.dup
begin
body.force_encoding(encoding)
rescue ArgumentError
body.force_encoding(Encoding::ISO_8859_1)
end
end
body.encode(Encoding::UTF_8)
end
end
end
end
require_relative "gumbo"