From eac2f9f6dd445496f83696a2e72dfeba0e8b54be Mon Sep 17 00:00:00 2001 From: Tim Aitken Date: Sat, 16 Sep 2023 09:24:27 -0700 Subject: [PATCH 1/7] implement hex-reader option --- include/json/reader.h | 2 ++ src/lib_json/json_reader.cpp | 63 ++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/include/json/reader.h b/include/json/reader.h index 46975d86f..9898197b3 100644 --- a/include/json/reader.h +++ b/include/json/reader.h @@ -324,6 +324,8 @@ class JSON_API CharReaderBuilder : public CharReader::Factory { * - `"allowSpecialFloats": false or true` * - If true, special float values (NaNs and infinities) are allowed and * their values are lossfree restorable. + * - `"allowHexadecimal": false or true` + * - If true, allow hexadecimal (eg 0xFFFF) to be used as unsigned integers. * - `"skipBom": false or true` * - If true, if the input starts with the Unicode byte order mark (BOM), * it is skipped. diff --git a/src/lib_json/json_reader.cpp b/src/lib_json/json_reader.cpp index 1ac5e81ab..f2bc3c264 100644 --- a/src/lib_json/json_reader.cpp +++ b/src/lib_json/json_reader.cpp @@ -875,6 +875,7 @@ class OurFeatures { bool failIfExtra_; bool rejectDupKeys_; bool allowSpecialFloats_; + bool allowHexadecimal_; bool skipBom_; size_t stackLimit_; }; // OurFeatures @@ -914,6 +915,7 @@ class OurReader { tokenArrayEnd, tokenString, tokenNumber, + tokenHexadecimal, tokenTrue, tokenFalse, tokenNull, @@ -952,11 +954,14 @@ class OurReader { bool readString(); bool readStringSingleQuote(); bool readNumber(bool checkInf); + bool readHexadecimal(); bool readValue(); bool readObject(Token& token); bool readArray(Token& token); bool decodeNumber(Token& token); bool decodeNumber(Token& token, Value& decoded); + bool decodeHexadecimal(Token& token); + bool decodeHexadecimal(Token& token, Value& decoded); bool decodeString(Token& token); bool decodeString(Token& token, String& decoded); bool decodeDouble(Token& token); @@ -1191,6 +1196,12 @@ bool OurReader::readToken(Token& token) { ok = readComment(); break; case '0': + if(match("x", 1)) { + token.type_ = tokenHexadecimal; + ok = features_.allowHexadecimal_; + readHexadecimal(); + break; + } case '1': case '2': case '3': @@ -1419,6 +1430,18 @@ bool OurReader::readNumber(bool checkInf) { } return true; } + +bool OurReader::readHexadecimal(void) { + Location p = current_; + char c = '0'; // stopgap for already consumed character + // integral part + while ((c >= '0' && c <= '9') + || (c >= 'a' && c <= 'f') + || (c >= 'A' && c <= 'F')) + c = (current_ = p) < end_ ? *p++ : '\0'; + return true; +} + bool OurReader::readString() { Char c = 0; while (current_ != end_) { @@ -1639,6 +1662,44 @@ bool OurReader::decodeNumber(Token& token, Value& decoded) { return true; } +bool OurReader::decodeHexadecimal(Token& token) { + Value decoded; + if (!decodeHexadecimal(token, decoded)) + return false; + currentValue().swapPayload(decoded); + currentValue().setOffsetStart(token.start_ - begin_); + currentValue().setOffsetLimit(token.end_ - begin_); + return true; +} + +bool OurReader::decodeHexadecimal(Token& token, Value& decoded) { + Json::LargestUInt value = 0; + constexpr Json::LargestUInt top = + Json::LargestUInt(0xF) << (sizeof(top) * 8) - 4; + + Location current = token.start_; + while (current < token.end_) { + Char c = *current++; + static_assert('A' < 'a'); + static_assert('0' < 'A'); + if (c == 'x') + continue; + else if (c >= 'a') + c -= 'a' - 10; + else if (c >= 'A') + c -= 'A' - 10; + else if (c >= '0') + c -= '0'; + else return addError( + "Contains non-hexadecimal digits.", token, current); + if (value & top) return addError( + "Number is too large for unsigned integer.", token, current); + value = value << 4 | static_cast(c); + } + decoded = value; + return true; +} + bool OurReader::decodeDouble(Token& token) { Value decoded; if (!decodeDouble(token, decoded)) @@ -1908,6 +1969,7 @@ CharReader* CharReaderBuilder::newCharReader() const { features.failIfExtra_ = settings_["failIfExtra"].asBool(); features.rejectDupKeys_ = settings_["rejectDupKeys"].asBool(); features.allowSpecialFloats_ = settings_["allowSpecialFloats"].asBool(); + features.allowHexadecimal_ = settings_["allowHexacecimal"].asBool(); features.skipBom_ = settings_["skipBom"].asBool(); return new OurCharReader(collectComments, features); } @@ -1925,6 +1987,7 @@ bool CharReaderBuilder::validate(Json::Value* invalid) const { "failIfExtra", "rejectDupKeys", "allowSpecialFloats", + "allowHexacecimal", "skipBom", }; for (auto si = settings_.begin(); si != settings_.end(); ++si) { From de2778d9390b792421a965514b3edf8dd2149a7a Mon Sep 17 00:00:00 2001 From: Tim Aitken Date: Sat, 16 Sep 2023 10:02:35 -0700 Subject: [PATCH 2/7] fix compiler warnings --- .gitignore | 1 + src/lib_json/json_reader.cpp | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 9682782fa..6aa96998b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ /doc/doxyfile /dist/ /.cache/ +/.vscode/ # MSVC project files: *.sln diff --git a/src/lib_json/json_reader.cpp b/src/lib_json/json_reader.cpp index f2bc3c264..d917873be 100644 --- a/src/lib_json/json_reader.cpp +++ b/src/lib_json/json_reader.cpp @@ -1200,8 +1200,12 @@ bool OurReader::readToken(Token& token) { token.type_ = tokenHexadecimal; ok = features_.allowHexadecimal_; readHexadecimal(); - break; } + else { + token.type_ = tokenNumber; + readNumber(false); + } + break; case '1': case '2': case '3': @@ -1675,16 +1679,12 @@ bool OurReader::decodeHexadecimal(Token& token) { bool OurReader::decodeHexadecimal(Token& token, Value& decoded) { Json::LargestUInt value = 0; constexpr Json::LargestUInt top = - Json::LargestUInt(0xF) << (sizeof(top) * 8) - 4; + Json::LargestUInt(0xF) << ((sizeof(top) * 8) - 4); - Location current = token.start_; + Location current = token.start_ + 2; while (current < token.end_) { Char c = *current++; - static_assert('A' < 'a'); - static_assert('0' < 'A'); - if (c == 'x') - continue; - else if (c >= 'a') + if (c >= 'a') c -= 'a' - 10; else if (c >= 'A') c -= 'A' - 10; From b69334ba82a6341cb657f7d76ceb1037886e4579 Mon Sep 17 00:00:00 2001 From: Tim Aitken Date: Sat, 16 Sep 2023 11:44:07 -0700 Subject: [PATCH 3/7] add me some unit tests --- src/test_lib_json/main.cpp | 81 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/src/test_lib_json/main.cpp b/src/test_lib_json/main.cpp index a6f21c45a..350f9a077 100644 --- a/src/test_lib_json/main.cpp +++ b/src/test_lib_json/main.cpp @@ -3556,6 +3556,87 @@ JSONTEST_FIXTURE_LOCAL(CharReaderAllowZeroesTest, issue176) { } } +struct CharReaderAllowHexadecimal : JsonTest::TestCase {}; + +JSONTEST_FIXTURE_LOCAL(CharReaderAllowHexadecimal, disallowHex) { + Json::CharReaderBuilder b; + CharReaderPtr reader(b.newCharReader()); + Json::Value root; + Json::String errs; + { + char const doc[] = R"({"a": 0x01})"; + bool ok = reader->parse(doc, doc + std::strlen(doc), &root, &errs); + JSONTEST_ASSERT(!ok); + JSONTEST_ASSERT_STRING_EQUAL( + "* Line 1, Column 7\n" + " Syntax error: value, object or array expected.\n", + errs); + } +} + +JSONTEST_FIXTURE_LOCAL(CharReaderAllowHexadecimal, hexObject) { + Json::CharReaderBuilder b; + b.settings_["allowHexadecimal"] = true; + CharReaderPtr reader(b.newCharReader()); + { + Json::Value root; + Json::String errs; + char const doc[] = R"({ + "a":0x9, + "b":0xf, + "c":0xF + })"; + bool ok = reader->parse(doc, doc + std::strlen(doc), &root, &errs); + JSONTEST_ASSERT(ok); + JSONTEST_ASSERT_STRING_EQUAL("", errs); + JSONTEST_ASSERT_EQUAL(3u, root.size()); + JSONTEST_ASSERT_EQUAL(0x9u, root.get("a", 0).asUInt()); + JSONTEST_ASSERT_EQUAL(0xfu, root.get("b", 0).asUInt()); + JSONTEST_ASSERT_EQUAL(0xFu, root.get("c", 0).asUInt()); + } +} + +JSONTEST_FIXTURE_LOCAL(CharReaderAllowHexadecimal, hexNumbers) { + Json::CharReaderBuilder b; + b.settings_["allowHexadecimal"] = true; + CharReaderPtr reader(b.newCharReader()); + + struct TestData { + bool ok; + Json::String in; + Json::LargestUInt out; + }; + const TestData test_data[] = { + {true, "9", 9}, // regular number + {true, "0x00", 0x00}, // zero + {true, "0x0123456789", 0x0123456789}, // numeric hex + {true, "0xABCDEF", 0xABCDEF}, // uppercase-letter hex + {true, "0xabcdef", 0xabcdef}, // lowercase-letter hex + {false, "x", 0 }, // leading x + {false, "0xx", 0 }, // extra x + {false, "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", 0} // too long + }; + for (const auto& td : test_data) { + Json::Value root; + Json::String errs; + const char* c0 = td.in.c_str(); + const char* c1 = c0 + td.in.size(); + bool ok = reader->parse(c0, c1, &root, &errs); + JSONTEST_ASSERT_EQUAL(td.ok, ok); + if (td.ok) + { + JSONTEST_ASSERT_EQUAL(0u, errs.size()); + JSONTEST_ASSERT(root.isConvertibleTo(Json::ValueType::uintValue)); + if (root.isConvertibleTo(Json::ValueType::uintValue)) + JSONTEST_ASSERT_EQUAL(root.asLargestUInt(), td.out); + } + else + { + JSONTEST_ASSERT(errs.size() > 0); + } + } +} + struct CharReaderAllowSpecialFloatsTest : JsonTest::TestCase {}; JSONTEST_FIXTURE_LOCAL(CharReaderAllowSpecialFloatsTest, specialFloat) { From 18dc444caf559b461c3f83762be931e875652f3e Mon Sep 17 00:00:00 2001 From: Tim Aitken Date: Sat, 16 Sep 2023 12:14:56 -0700 Subject: [PATCH 4/7] get basic tests running --- src/lib_json/json_reader.cpp | 7 +++++-- src/test_lib_json/main.cpp | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/lib_json/json_reader.cpp b/src/lib_json/json_reader.cpp index d917873be..a43b9324b 100644 --- a/src/lib_json/json_reader.cpp +++ b/src/lib_json/json_reader.cpp @@ -1083,6 +1083,9 @@ bool OurReader::readValue() { case tokenNumber: successful = decodeNumber(token); break; + case tokenHexadecimal: + successful = decodeHexadecimal(token); + break; case tokenString: successful = decodeString(token); break; @@ -1969,7 +1972,7 @@ CharReader* CharReaderBuilder::newCharReader() const { features.failIfExtra_ = settings_["failIfExtra"].asBool(); features.rejectDupKeys_ = settings_["rejectDupKeys"].asBool(); features.allowSpecialFloats_ = settings_["allowSpecialFloats"].asBool(); - features.allowHexadecimal_ = settings_["allowHexacecimal"].asBool(); + features.allowHexadecimal_ = settings_["allowHexadecimal"].asBool(); features.skipBom_ = settings_["skipBom"].asBool(); return new OurCharReader(collectComments, features); } @@ -1987,7 +1990,7 @@ bool CharReaderBuilder::validate(Json::Value* invalid) const { "failIfExtra", "rejectDupKeys", "allowSpecialFloats", - "allowHexacecimal", + "allowHexadecimal", "skipBom", }; for (auto si = settings_.begin(); si != settings_.end(); ++si) { diff --git a/src/test_lib_json/main.cpp b/src/test_lib_json/main.cpp index 350f9a077..ac5fc4220 100644 --- a/src/test_lib_json/main.cpp +++ b/src/test_lib_json/main.cpp @@ -3577,6 +3577,8 @@ JSONTEST_FIXTURE_LOCAL(CharReaderAllowHexadecimal, disallowHex) { JSONTEST_FIXTURE_LOCAL(CharReaderAllowHexadecimal, hexObject) { Json::CharReaderBuilder b; b.settings_["allowHexadecimal"] = true; + Json::Value invalid; + JSONTEST_ASSERT(b.validate(&invalid)) << invalid; CharReaderPtr reader(b.newCharReader()); { Json::Value root; @@ -3622,7 +3624,7 @@ JSONTEST_FIXTURE_LOCAL(CharReaderAllowHexadecimal, hexNumbers) { const char* c0 = td.in.c_str(); const char* c1 = c0 + td.in.size(); bool ok = reader->parse(c0, c1, &root, &errs); - JSONTEST_ASSERT_EQUAL(td.ok, ok); + JSONTEST_ASSERT(td.ok == ok) << "in: " << td.in; if (td.ok) { JSONTEST_ASSERT_EQUAL(0u, errs.size()); From c48253682fce86589bc2cc96944d9a2f7feb1cf9 Mon Sep 17 00:00:00 2001 From: Tim Aitken Date: Sat, 16 Sep 2023 14:36:28 -0700 Subject: [PATCH 5/7] careful handling of long numbers --- src/lib_json/json_reader.cpp | 35 +++++++++++++++------------ src/test_lib_json/main.cpp | 47 +++++++++++++++++++++--------------- 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/src/lib_json/json_reader.cpp b/src/lib_json/json_reader.cpp index a43b9324b..0b4dde21a 100644 --- a/src/lib_json/json_reader.cpp +++ b/src/lib_json/json_reader.cpp @@ -1440,12 +1440,14 @@ bool OurReader::readNumber(bool checkInf) { bool OurReader::readHexadecimal(void) { Location p = current_; - char c = '0'; // stopgap for already consumed character - // integral part - while ((c >= '0' && c <= '9') - || (c >= 'a' && c <= 'f') - || (c >= 'A' && c <= 'F')) - c = (current_ = p) < end_ ? *p++ : '\0'; + for (; p < end_; ++p) + { + char c = *p; + if ( (c < '0' || c > '9') + && (c < 'a' || c > 'f') + && (c < 'A' || c > 'F') ) break; + } + current_ = p; return true; } @@ -1680,23 +1682,24 @@ bool OurReader::decodeHexadecimal(Token& token) { } bool OurReader::decodeHexadecimal(Token& token, Value& decoded) { + Location current = token.start_; + if (current < token.end_ && *current == '0') ++current; + if (current < token.end_ && *current == 'x') ++current; Json::LargestUInt value = 0; - constexpr Json::LargestUInt top = - Json::LargestUInt(0xF) << ((sizeof(top) * 8) - 4); - - Location current = token.start_ + 2; - while (current < token.end_) { - Char c = *current++; + if (current >= token.end_) + return addError("Zero hexadecimal digits.", token); + if (current + (sizeof(value) * 2) < token.end_) + return addError("Token too long to be unsigned integer.", token, current); + for (; current < token.end_; ++current) { + Char c = *current; if (c >= 'a') c -= 'a' - 10; else if (c >= 'A') c -= 'A' - 10; else if (c >= '0') c -= '0'; - else return addError( - "Contains non-hexadecimal digits.", token, current); - if (value & top) return addError( - "Number is too large for unsigned integer.", token, current); + else + return addError("Contains non-hexadecimal digits.", token, current); value = value << 4 | static_cast(c); } decoded = value; diff --git a/src/test_lib_json/main.cpp b/src/test_lib_json/main.cpp index ac5fc4220..b67609253 100644 --- a/src/test_lib_json/main.cpp +++ b/src/test_lib_json/main.cpp @@ -3564,7 +3564,7 @@ JSONTEST_FIXTURE_LOCAL(CharReaderAllowHexadecimal, disallowHex) { Json::Value root; Json::String errs; { - char const doc[] = R"({"a": 0x01})"; + char const doc[] = R"({ "a":0x9, "b":0xf, "c":0xF })"; bool ok = reader->parse(doc, doc + std::strlen(doc), &root, &errs); JSONTEST_ASSERT(!ok); JSONTEST_ASSERT_STRING_EQUAL( @@ -3583,11 +3583,7 @@ JSONTEST_FIXTURE_LOCAL(CharReaderAllowHexadecimal, hexObject) { { Json::Value root; Json::String errs; - char const doc[] = R"({ - "a":0x9, - "b":0xf, - "c":0xF - })"; + char const doc[] = R"({ "a":0x9, "b":0xf, "c":0xF })"; bool ok = reader->parse(doc, doc + std::strlen(doc), &root, &errs); JSONTEST_ASSERT(ok); JSONTEST_ASSERT_STRING_EQUAL("", errs); @@ -3605,18 +3601,31 @@ JSONTEST_FIXTURE_LOCAL(CharReaderAllowHexadecimal, hexNumbers) { struct TestData { bool ok; - Json::String in; Json::LargestUInt out; + Json::String in; }; + constexpr int _ = 0; // ignored const TestData test_data[] = { - {true, "9", 9}, // regular number - {true, "0x00", 0x00}, // zero - {true, "0x0123456789", 0x0123456789}, // numeric hex - {true, "0xABCDEF", 0xABCDEF}, // uppercase-letter hex - {true, "0xabcdef", 0xabcdef}, // lowercase-letter hex - {false, "x", 0 }, // leading x - {false, "0xx", 0 }, // extra x - {false, "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", 0} // too long + {true, 99, "99"}, // regular number + {true, 0x99, "0x99"}, // hexadecimal number + {false, _, "AA"}, // missing prefix + {false, _, "xAA"}, // partial prefix + {true, 0xAA, "0xAA"}, // with prefix + {true, 0x00, "0x00"}, // zero + {true, 0x0123456789, "0x0123456789"}, // numeric hex + {true, 0xABCDEF, "0xABCDEF"}, // uppercase-letter hex + {true, 0xabcdef, "0xabcdef"}, // lowercase-letter hex +#ifdef JSON_HAS_INT64 + {true, 0xFfffFfffFfffFfff, "0xFfffFfffFfffFfff"}, // max + {false, _, "0x1FfffFfffFfffFfff"}, // too long +#else + {true, 0xFfffFfff, "0xFfffFfff"}, // max + {false, _, "0x1FfffFfff"}, // too long +#endif + {false, _, "0x000000000000000000000000000000000000000"}, // too long + {false, _, "x"}, // leading x + {false, _, "0x"}, // empty number + {false, _, "0xx"} // extra x }; for (const auto& td : test_data) { Json::Value root; @@ -3627,10 +3636,10 @@ JSONTEST_FIXTURE_LOCAL(CharReaderAllowHexadecimal, hexNumbers) { JSONTEST_ASSERT(td.ok == ok) << "in: " << td.in; if (td.ok) { - JSONTEST_ASSERT_EQUAL(0u, errs.size()); - JSONTEST_ASSERT(root.isConvertibleTo(Json::ValueType::uintValue)); - if (root.isConvertibleTo(Json::ValueType::uintValue)) - JSONTEST_ASSERT_EQUAL(root.asLargestUInt(), td.out); + JSONTEST_ASSERT_STRING_EQUAL("", errs); + JSONTEST_ASSERT(root.isUInt64()); + if (root.isUInt64()) + JSONTEST_ASSERT_EQUAL(td.out, root.asLargestUInt()); } else { From a672fc1cec99d86871734b9ff3c310b17d144e63 Mon Sep 17 00:00:00 2001 From: Tim Aitken Date: Sat, 16 Sep 2023 20:17:05 -0700 Subject: [PATCH 6/7] version bump --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8920544a3..92a4d5e18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,7 +72,7 @@ project(jsoncpp # 2. ./include/json/version.h # 3. ./CMakeLists.txt # IMPORTANT: also update the PROJECT_SOVERSION!! - VERSION 1.9.5 # [.[.[.]]] + VERSION 1.10.0 # [.[.[.]]] LANGUAGES CXX) message(STATUS "JsonCpp Version: ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}") From 7a74a996f3892f04bb33c052e28b9237fdabf45a Mon Sep 17 00:00:00 2001 From: Tim Aitken Date: Tue, 20 Feb 2024 12:40:59 -0800 Subject: [PATCH 7/7] add upper-bounds check for valid hex characters --- src/lib_json/json_reader.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib_json/json_reader.cpp b/src/lib_json/json_reader.cpp index 0b4dde21a..00c051fc7 100644 --- a/src/lib_json/json_reader.cpp +++ b/src/lib_json/json_reader.cpp @@ -1692,11 +1692,11 @@ bool OurReader::decodeHexadecimal(Token& token, Value& decoded) { return addError("Token too long to be unsigned integer.", token, current); for (; current < token.end_; ++current) { Char c = *current; - if (c >= 'a') + if (c >= 'a' && c <= 'f') c -= 'a' - 10; - else if (c >= 'A') + else if (c >= 'A' && c <= 'F') c -= 'A' - 10; - else if (c >= '0') + else if (c >= '0' && c <= '9') c -= '0'; else return addError("Contains non-hexadecimal digits.", token, current);