diff --git a/python-package/xgboost/core.py b/python-package/xgboost/core.py
index 8ae45c5bc..802e93a60 100644
--- a/python-package/xgboost/core.py
+++ b/python-package/xgboost/core.py
@@ -363,7 +363,9 @@ class DMatrix(object):
                 nthread))
 
     def __del__(self):
-        _check_call(_LIB.XGDMatrixFree(self.handle))
+        if self.handle is not None:
+            _check_call(_LIB.XGDMatrixFree(self.handle))
+            self.handle = None
 
     def get_float_info(self, field):
         """Get float property from the DMatrix.
@@ -738,7 +740,9 @@ class Booster(object):
             self.load_model(model_file)
 
     def __del__(self):
-        _LIB.XGBoosterFree(self.handle)
+        if self.handle is not None:
+            _check_call(_LIB.XGBoosterFree(self.handle))
+            self.handle = None
 
     def __getstate__(self):
         # can't pickle ctypes pointers