Add test for invalid categorical data values. (#7380)

* Add test for invalid categorical data values.

* Add check during sketching.

tests/python-gpu/test_gpu_updaters.py

@@ -95,6 +95,21 @@ class TestGPUUpdaters:
         rounds = 4
         self.run_categorical_basic(rows, cols, rounds, cats)
 
+    def test_invalid_categorical(self):
+        import cupy as cp
+        rng = np.random.default_rng()
+        X = rng.normal(loc=0, scale=1, size=1000).reshape(100, 10)
+        y = rng.normal(loc=0, scale=1, size=100)
+
+        # Check is performe during sketching.
+        Xy = xgb.DMatrix(X, y, feature_types=["c"] * 10)
+        with pytest.raises(ValueError):
+            xgb.train({"tree_method": "gpu_hist"}, Xy)
+
+        X, y = cp.array(X), cp.array(y)
+        with pytest.raises(ValueError):
+            Xy = xgb.DeviceQuantileDMatrix(X, y, feature_types=["c"] * 10)
+
     @pytest.mark.skipif(**tm.no_cupy())
     @given(parameter_strategy, strategies.integers(1, 20),
            tm.dataset_strategy)