diff --git a/src/bin/pg_upgrade/.gitignore b/src/bin/pg_upgrade/.gitignore
index 939e50db6c..05200a09f1 100644
--- a/src/bin/pg_upgrade/.gitignore
+++ b/src/bin/pg_upgrade/.gitignore
@@ -1,7 +1,4 @@
 /pg_upgrade
 # Generated by test suite
-/delete_old_cluster.sh
-/delete_old_cluster.bat
-/reindex_hash.sql
 /log/
 /tmp_check/
diff --git a/src/bin/pg_upgrade/Makefile b/src/bin/pg_upgrade/Makefile
index 587793e815..80ebe9bd17 100644
--- a/src/bin/pg_upgrade/Makefile
+++ b/src/bin/pg_upgrade/Makefile
@@ -48,8 +48,7 @@ uninstall:
 
 clean distclean maintainer-clean:
 	rm -f pg_upgrade$(X) $(OBJS)
-	rm -rf delete_old_cluster.sh log/ tmp_check/ \
-	       reindex_hash.sql
+	rm -rf log/ tmp_check/
 
 check:
 	$(prove_check)
diff --git a/src/bin/pg_upgrade/t/002_pg_upgrade.pl b/src/bin/pg_upgrade/t/002_pg_upgrade.pl
index 80437e93b7..55c7354ba2 100644
--- a/src/bin/pg_upgrade/t/002_pg_upgrade.pl
+++ b/src/bin/pg_upgrade/t/002_pg_upgrade.pl
@@ -206,6 +206,11 @@ if (defined($ENV{oldinstall}))
 	}
 }
 
+# In a VPATH build, we'll be started in the source directory, but we want
+# to run pg_upgrade in the build directory so that any files generated finish
+# in it, like delete_old_cluster.{sh,bat}.
+chdir ${PostgreSQL::Test::Utils::tmp_check};
+
 # Upgrade the instance.
 $oldnode->stop;
 command_ok(
@@ -238,7 +243,7 @@ $newnode->command_ok(
 		'-d',         $newnode->connstr('postgres'),
 		'-f',         "$tempdir/dump2.sql"
 	],
-	'dump before running pg_upgrade');
+	'dump after running pg_upgrade');
 
 # Compare the two dumps, there should be no differences.
 my $compare_res = compare("$tempdir/dump1.sql", "$tempdir/dump2.sql");