input_adapters.hpp 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374
  1. #pragma once
  2. #include <cassert> // assert
  3. #include <cstddef> // size_t
  4. #include <cstring> // strlen
  5. #include <istream> // istream
  6. #include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next
  7. #include <memory> // shared_ptr, make_shared, addressof
  8. #include <numeric> // accumulate
  9. #include <string> // string, char_traits
  10. #include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer
  11. #include <utility> // pair, declval
  12. #include <nlohmann/detail/macro_scope.hpp>
  13. namespace nlohmann
  14. {
  15. namespace detail
  16. {
  17. /// the supported input formats
  18. enum class input_format_t { json, cbor, msgpack, ubjson };
  19. ////////////////////
  20. // input adapters //
  21. ////////////////////
  22. /*!
  23. @brief abstract input adapter interface
  24. Produces a stream of std::char_traits<char>::int_type characters from a
  25. std::istream, a buffer, or some other input type. Accepts the return of
  26. exactly one non-EOF character for future input. The int_type characters
  27. returned consist of all valid char values as positive values (typically
  28. unsigned char), plus an EOF value outside that range, specified by the value
  29. of the function std::char_traits<char>::eof(). This value is typically -1, but
  30. could be any arbitrary value which is not a valid char value.
  31. */
  32. struct input_adapter_protocol
  33. {
  34. /// get a character [0,255] or std::char_traits<char>::eof().
  35. virtual std::char_traits<char>::int_type get_character() = 0;
  36. virtual ~input_adapter_protocol() = default;
  37. };
  38. /// a type to simplify interfaces
  39. using input_adapter_t = std::shared_ptr<input_adapter_protocol>;
  40. /*!
  41. Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at
  42. beginning of input. Does not support changing the underlying std::streambuf
  43. in mid-input. Maintains underlying std::istream and std::streambuf to support
  44. subsequent use of standard std::istream operations to process any input
  45. characters following those used in parsing the JSON input. Clears the
  46. std::istream flags; any input errors (e.g., EOF) will be detected by the first
  47. subsequent call for input from the std::istream.
  48. */
  49. class input_stream_adapter : public input_adapter_protocol
  50. {
  51. public:
  52. ~input_stream_adapter() override
  53. {
  54. // clear stream flags; we use underlying streambuf I/O, do not
  55. // maintain ifstream flags
  56. is.clear();
  57. }
  58. explicit input_stream_adapter(std::istream& i)
  59. : is(i), sb(*i.rdbuf())
  60. {}
  61. // delete because of pointer members
  62. input_stream_adapter(const input_stream_adapter&) = delete;
  63. input_stream_adapter& operator=(input_stream_adapter&) = delete;
  64. // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to
  65. // ensure that std::char_traits<char>::eof() and the character 0xFF do not
  66. // end up as the same value, eg. 0xFFFFFFFF.
  67. std::char_traits<char>::int_type get_character() override
  68. {
  69. return sb.sbumpc();
  70. }
  71. private:
  72. /// the associated input stream
  73. std::istream& is;
  74. std::streambuf& sb;
  75. };
  76. /// input adapter for buffer input
  77. class input_buffer_adapter : public input_adapter_protocol
  78. {
  79. public:
  80. input_buffer_adapter(const char* b, const std::size_t l)
  81. : cursor(b), limit(b + l)
  82. {}
  83. // delete because of pointer members
  84. input_buffer_adapter(const input_buffer_adapter&) = delete;
  85. input_buffer_adapter& operator=(input_buffer_adapter&) = delete;
  86. std::char_traits<char>::int_type get_character() noexcept override
  87. {
  88. if (JSON_LIKELY(cursor < limit))
  89. {
  90. return std::char_traits<char>::to_int_type(*(cursor++));
  91. }
  92. return std::char_traits<char>::eof();
  93. }
  94. private:
  95. /// pointer to the current character
  96. const char* cursor;
  97. /// pointer past the last character
  98. const char* const limit;
  99. };
  100. template<typename WideStringType>
  101. class wide_string_input_adapter : public input_adapter_protocol
  102. {
  103. public:
  104. explicit wide_string_input_adapter(const WideStringType& w) : str(w) {}
  105. std::char_traits<char>::int_type get_character() noexcept override
  106. {
  107. // check if buffer needs to be filled
  108. if (utf8_bytes_index == utf8_bytes_filled)
  109. {
  110. if (sizeof(typename WideStringType::value_type) == 2)
  111. {
  112. fill_buffer_utf16();
  113. }
  114. else
  115. {
  116. fill_buffer_utf32();
  117. }
  118. assert(utf8_bytes_filled > 0);
  119. assert(utf8_bytes_index == 0);
  120. }
  121. // use buffer
  122. assert(utf8_bytes_filled > 0);
  123. assert(utf8_bytes_index < utf8_bytes_filled);
  124. return utf8_bytes[utf8_bytes_index++];
  125. }
  126. private:
  127. void fill_buffer_utf16()
  128. {
  129. utf8_bytes_index = 0;
  130. if (current_wchar == str.size())
  131. {
  132. utf8_bytes[0] = std::char_traits<char>::eof();
  133. utf8_bytes_filled = 1;
  134. }
  135. else
  136. {
  137. // get the current character
  138. const int wc = static_cast<int>(str[current_wchar++]);
  139. // UTF-16 to UTF-8 encoding
  140. if (wc < 0x80)
  141. {
  142. utf8_bytes[0] = wc;
  143. utf8_bytes_filled = 1;
  144. }
  145. else if (wc <= 0x7FF)
  146. {
  147. utf8_bytes[0] = 0xC0 | ((wc >> 6));
  148. utf8_bytes[1] = 0x80 | (wc & 0x3F);
  149. utf8_bytes_filled = 2;
  150. }
  151. else if (0xD800 > wc or wc >= 0xE000)
  152. {
  153. utf8_bytes[0] = 0xE0 | ((wc >> 12));
  154. utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F);
  155. utf8_bytes[2] = 0x80 | (wc & 0x3F);
  156. utf8_bytes_filled = 3;
  157. }
  158. else
  159. {
  160. if (current_wchar < str.size())
  161. {
  162. const int wc2 = static_cast<int>(str[current_wchar++]);
  163. const int charcode = 0x10000 + (((wc & 0x3FF) << 10) | (wc2 & 0x3FF));
  164. utf8_bytes[0] = 0xf0 | (charcode >> 18);
  165. utf8_bytes[1] = 0x80 | ((charcode >> 12) & 0x3F);
  166. utf8_bytes[2] = 0x80 | ((charcode >> 6) & 0x3F);
  167. utf8_bytes[3] = 0x80 | (charcode & 0x3F);
  168. utf8_bytes_filled = 4;
  169. }
  170. else
  171. {
  172. // unknown character
  173. ++current_wchar;
  174. utf8_bytes[0] = wc;
  175. utf8_bytes_filled = 1;
  176. }
  177. }
  178. }
  179. }
  180. void fill_buffer_utf32()
  181. {
  182. utf8_bytes_index = 0;
  183. if (current_wchar == str.size())
  184. {
  185. utf8_bytes[0] = std::char_traits<char>::eof();
  186. utf8_bytes_filled = 1;
  187. }
  188. else
  189. {
  190. // get the current character
  191. const int wc = static_cast<int>(str[current_wchar++]);
  192. // UTF-32 to UTF-8 encoding
  193. if (wc < 0x80)
  194. {
  195. utf8_bytes[0] = wc;
  196. utf8_bytes_filled = 1;
  197. }
  198. else if (wc <= 0x7FF)
  199. {
  200. utf8_bytes[0] = 0xC0 | ((wc >> 6) & 0x1F);
  201. utf8_bytes[1] = 0x80 | (wc & 0x3F);
  202. utf8_bytes_filled = 2;
  203. }
  204. else if (wc <= 0xFFFF)
  205. {
  206. utf8_bytes[0] = 0xE0 | ((wc >> 12) & 0x0F);
  207. utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F);
  208. utf8_bytes[2] = 0x80 | (wc & 0x3F);
  209. utf8_bytes_filled = 3;
  210. }
  211. else if (wc <= 0x10FFFF)
  212. {
  213. utf8_bytes[0] = 0xF0 | ((wc >> 18 ) & 0x07);
  214. utf8_bytes[1] = 0x80 | ((wc >> 12) & 0x3F);
  215. utf8_bytes[2] = 0x80 | ((wc >> 6) & 0x3F);
  216. utf8_bytes[3] = 0x80 | (wc & 0x3F);
  217. utf8_bytes_filled = 4;
  218. }
  219. else
  220. {
  221. // unknown character
  222. utf8_bytes[0] = wc;
  223. utf8_bytes_filled = 1;
  224. }
  225. }
  226. }
  227. private:
  228. /// the wstring to process
  229. const WideStringType& str;
  230. /// index of the current wchar in str
  231. std::size_t current_wchar = 0;
  232. /// a buffer for UTF-8 bytes
  233. std::array<std::char_traits<char>::int_type, 4> utf8_bytes = {{0, 0, 0, 0}};
  234. /// index to the utf8_codes array for the next valid byte
  235. std::size_t utf8_bytes_index = 0;
  236. /// number of valid bytes in the utf8_codes array
  237. std::size_t utf8_bytes_filled = 0;
  238. };
  239. class input_adapter
  240. {
  241. public:
  242. // native support
  243. /// input adapter for input stream
  244. input_adapter(std::istream& i)
  245. : ia(std::make_shared<input_stream_adapter>(i)) {}
  246. /// input adapter for input stream
  247. input_adapter(std::istream&& i)
  248. : ia(std::make_shared<input_stream_adapter>(i)) {}
  249. input_adapter(const std::wstring& ws)
  250. : ia(std::make_shared<wide_string_input_adapter<std::wstring>>(ws)) {}
  251. input_adapter(const std::u16string& ws)
  252. : ia(std::make_shared<wide_string_input_adapter<std::u16string>>(ws)) {}
  253. input_adapter(const std::u32string& ws)
  254. : ia(std::make_shared<wide_string_input_adapter<std::u32string>>(ws)) {}
  255. /// input adapter for buffer
  256. template<typename CharT,
  257. typename std::enable_if<
  258. std::is_pointer<CharT>::value and
  259. std::is_integral<typename std::remove_pointer<CharT>::type>::value and
  260. sizeof(typename std::remove_pointer<CharT>::type) == 1,
  261. int>::type = 0>
  262. input_adapter(CharT b, std::size_t l)
  263. : ia(std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(b), l)) {}
  264. // derived support
  265. /// input adapter for string literal
  266. template<typename CharT,
  267. typename std::enable_if<
  268. std::is_pointer<CharT>::value and
  269. std::is_integral<typename std::remove_pointer<CharT>::type>::value and
  270. sizeof(typename std::remove_pointer<CharT>::type) == 1,
  271. int>::type = 0>
  272. input_adapter(CharT b)
  273. : input_adapter(reinterpret_cast<const char*>(b),
  274. std::strlen(reinterpret_cast<const char*>(b))) {}
  275. /// input adapter for iterator range with contiguous storage
  276. template<class IteratorType,
  277. typename std::enable_if<
  278. std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value,
  279. int>::type = 0>
  280. input_adapter(IteratorType first, IteratorType last)
  281. {
  282. // assertion to check that the iterator range is indeed contiguous,
  283. // see http://stackoverflow.com/a/35008842/266378 for more discussion
  284. assert(std::accumulate(
  285. first, last, std::pair<bool, int>(true, 0),
  286. [&first](std::pair<bool, int> res, decltype(*first) val)
  287. {
  288. res.first &= (val == *(std::next(std::addressof(*first), res.second++)));
  289. return res;
  290. }).first);
  291. // assertion to check that each element is 1 byte long
  292. static_assert(
  293. sizeof(typename std::iterator_traits<IteratorType>::value_type) == 1,
  294. "each element in the iterator range must have the size of 1 byte");
  295. const auto len = static_cast<size_t>(std::distance(first, last));
  296. if (JSON_LIKELY(len > 0))
  297. {
  298. // there is at least one element: use the address of first
  299. ia = std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(&(*first)), len);
  300. }
  301. else
  302. {
  303. // the address of first cannot be used: use nullptr
  304. ia = std::make_shared<input_buffer_adapter>(nullptr, len);
  305. }
  306. }
  307. /// input adapter for array
  308. template<class T, std::size_t N>
  309. input_adapter(T (&array)[N])
  310. : input_adapter(std::begin(array), std::end(array)) {}
  311. /// input adapter for contiguous container
  312. template<class ContiguousContainer, typename
  313. std::enable_if<not std::is_pointer<ContiguousContainer>::value and
  314. std::is_base_of<std::random_access_iterator_tag, typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value,
  315. int>::type = 0>
  316. input_adapter(const ContiguousContainer& c)
  317. : input_adapter(std::begin(c), std::end(c)) {}
  318. operator input_adapter_t()
  319. {
  320. return ia;
  321. }
  322. private:
  323. /// the actual adapter
  324. input_adapter_t ia = nullptr;
  325. };
  326. }
  327. }