diff --git a/doc/tutorials/saving_model.rst b/doc/tutorials/saving_model.rst
index 7d416ccb1..a1d3d8ac5 100644
--- a/doc/tutorials/saving_model.rst
+++ b/doc/tutorials/saving_model.rst
@@ -195,7 +195,9 @@ You can load it back to the model generated by same version of XGBoost by:
 
   bst.load_config(config)
 
-This way users can study the internal representation more closely.
+This way users can study the internal representation more closely.  Please note that some
+JSON generators make use of locale dependent floating point serialization methods, which
+is not supported by XGBoost.
 
 ************
 Future Plans
diff --git a/src/common/json.cc b/src/common/json.cc
index 52878bbf9..da0d5c012 100644
--- a/src/common/json.cc
+++ b/src/common/json.cc
@@ -2,6 +2,7 @@
  * Copyright (c) by Contributors 2019
  */
 #include <cctype>
+#include <locale>
 #include <sstream>
 #include <limits>
 #include <cmath>
@@ -692,47 +693,23 @@ Json JsonReader::ParseBoolean() {
   return Json{JsonBoolean{result}};
 }
 
-// This is an ad-hoc solution for writing numeric value in standard way.  We need to add
-// a locale independent way of writing stream like `std::{from, to}_chars' from C++-17.
-// FIXME(trivialfis): Remove this.
-class GlobalCLocale {
-  std::locale ori_;
-
- public:
-  GlobalCLocale() : ori_{std::locale()} {
-    std::string const name {"C"};
-    try {
-      std::locale::global(std::locale(name.c_str()));
-    } catch (std::runtime_error const& e) {
-      LOG(FATAL) << "Failed to set locale: " << name;
-    }
-  }
-  ~GlobalCLocale() {
-    std::locale::global(ori_);
-  }
-};
-
 Json Json::Load(StringView str) {
-  GlobalCLocale guard;
   JsonReader reader(str);
   Json json{reader.Load()};
   return json;
 }
 
 Json Json::Load(JsonReader* reader) {
-  GlobalCLocale guard;
   Json json{reader->Load()};
   return json;
 }
 
 void Json::Dump(Json json, std::ostream *stream, bool pretty) {
-  GlobalCLocale guard;
   JsonWriter writer(stream, pretty);
   writer.Save(json);
 }
 
 void Json::Dump(Json json, std::string* str, bool pretty) {
-  GlobalCLocale guard;
   std::stringstream ss;
   JsonWriter writer(&ss, pretty);
   writer.Save(json);
diff --git a/tests/python/test_basic_models.py b/tests/python/test_basic_models.py
index a5eb395df..9d947f1f4 100644
--- a/tests/python/test_basic_models.py
+++ b/tests/python/test_basic_models.py
@@ -5,6 +5,7 @@ import os
 import json
 import testing as tm
 import pytest
+import locale
 
 dpath = 'demo/data/'
 dtrain = xgb.DMatrix(dpath + 'agaricus.txt.train')
@@ -300,6 +301,7 @@ class TestModels(unittest.TestCase):
             'reg_loss_param']['scale_pos_weight']) == 0.5
 
     def test_model_json_io(self):
+        loc = locale.getpreferredencoding(False)
         model_path = 'test_model_json_io.json'
         parameters = {'tree_method': 'hist', 'booster': 'gbtree'}
         j_model = json_model(model_path, parameters)
@@ -313,6 +315,7 @@ class TestModels(unittest.TestCase):
         assert isinstance(j_model['learner'], dict)
 
         os.remove(model_path)
+        assert locale.getpreferredencoding(False) == loc
 
     @pytest.mark.skipif(**tm.no_json_schema())
     def test_json_schema(self):