diff --git a/Jenkinsfile b/Jenkinsfile
index 274caeb3e..60e8116f3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -38,12 +38,15 @@ pipeline {
       agent { label 'job_initializer' }
       steps {
         script {
+          def buildNumber = env.BUILD_NUMBER as int
+          if (buildNumber > 1) milestone(buildNumber - 1)
+          milestone(buildNumber)
+
           checkoutSrcs()
           commit_id = "${GIT_COMMIT}"
         }
         sh 'python3 tests/jenkins_get_approval.py'
         stash name: 'srcs'
-        milestone ordinal: 1
       }
     }
     stage('Jenkins Linux: Formatting Check') {
@@ -57,7 +60,6 @@ pipeline {
             'doxygen': { Doxygen() }
           ])
         }
-        milestone ordinal: 2
       }
     }
     stage('Jenkins Linux: Build') {
@@ -80,7 +82,6 @@ pipeline {
             'build-jvm-doc': { BuildJVMDoc() }
           ])
         }
-        milestone ordinal: 3
       }
     }
     stage('Jenkins Linux: Test') {
@@ -103,7 +104,6 @@ pipeline {
             'test-r-3.5.3': { TestR(use_r35: true) }
           ])
         }
-        milestone ordinal: 4
       }
     }
     stage('Jenkins Linux: Deploy') {
@@ -114,7 +114,6 @@ pipeline {
             'deploy-jvm-packages': { DeployJVMPackages(spark_version: '3.0.0') }
           ])
         }
-        milestone ordinal: 5
       }
     }
   }
diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64
index 3f4611404..573a900a7 100644
--- a/Jenkinsfile-win64
+++ b/Jenkinsfile-win64
@@ -25,12 +25,14 @@ pipeline {
       agent { label 'job_initializer' }
       steps {
         script {
+          def buildNumber = env.BUILD_NUMBER as int
+          if (buildNumber > 1) milestone(buildNumber - 1)
+          milestone(buildNumber)
           checkoutSrcs()
           commit_id = "${GIT_COMMIT}"
         }
         sh 'python3 tests/jenkins_get_approval.py'
         stash name: 'srcs'
-        milestone ordinal: 1
       }
     }
     stage('Jenkins Win64: Build') {
@@ -41,7 +43,6 @@ pipeline {
             'build-win64-cuda10.1': { BuildWin64() }
           ])
         }
-        milestone ordinal: 2
       }
     }
     stage('Jenkins Win64: Test') {
@@ -52,7 +53,6 @@ pipeline {
             'test-win64-cuda10.1': { TestWin64() },
           ])
         }
-        milestone ordinal: 3
       }
     }
   }