From 21f8596c58a9791e7ea1f4ba93be17eed60a05b6 Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Tue, 9 Jul 2013 13:09:31 +0100 Subject: tests: Regression testsuite A number of Syslinux releases have contained regressions when compared with previous versions. Now that the size of the derivative grid has exploded (PXELINUX, SYSLINUX,.... bios, efi32, efi64) we need to have some means of ensuring we don't introduce regressions so easily. Start building a suite of regression tests. Regression tests should test end-to-end functionality, e.g. loading a Linux kernel. If a test fails, that failure represents a regression in some behaviour, e.g. Syslinux behaves differently than it previously did. The idea is that when bugs are reported in Syslinux, we can create new tests that reproduce the buggy behaviour. Once a fix has been committed the new test should pass. From that point forward, the test provides a means of ensuring we never reintroduce that bug. The collection of tests illustrates the agreed upon behaviour of Syslinux. To execute the regression testsuite type, make regression Sample output might look like, Executing regression tests for SYSLINUX Running Linux kernel regression tests... [+] empty passed [+] kernelhello passed Running COM32 module regression tests... [+] chaindisk passed Executing regression tests for PXELINUX Running Linux kernel regression tests... [!] empty failed [+] kernelhello passed [+] pxetest passed Qemu is used to execute the regression tests in a virtual environment. Because it takes time to load Qemu, we can forgive the regression testsuite for not executing instantaneously. However, developers should keep in mind the idea of "failing fast" in their regression tests to minimize execution time - that is, if a test is going to fail, make it fail as quickly as possible to reduce the testrun time. It's possible to have derivative-specific tests, where the test only runs for a specific Syslinux boot loader. For an example see pxetest in tests/linux/Makefile. Currently the regression testsuite requires root privileges to execute and makes several calls to sudo(8). In future it would be nice to get rid of this requirement. Signed-off-by: Matt Fleming --- Makefile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index a9b515c3..d90635af 100644 --- a/Makefile +++ b/Makefile @@ -72,7 +72,7 @@ include $(MAKEDIR)/syslinux.mk -include $(OBJDIR)/version.mk private-targets = prerel unprerel official release burn isolinux.iso \ - preupload upload + preupload upload test ifeq ($(MAKECMDGOALS),) MAKECMDGOALS += all @@ -101,6 +101,13 @@ $(filter-out $(private-targets), $(MAKECMDGOALS)): $(MAKE) -C $(OBJDIR) -f $(CURDIR)/Makefile SRC="$(topdir)" \ OBJ=$(OBJDIR) objdir=$(OBJDIR) $(MAKECMDGOALS) +test: + printf "Executing unit tests\n" + $(MAKE) -C com32/lib/syslinux/tests all + $(MAKE) -C tests SRC="$(topdir)/tests" OBJ="$(topdir)/tests" \ + objdir=$(OBJDIR) \ + -f $(topdir)/tests/Makefile all + # Hook to add private Makefile targets for the maintainer. -include $(topdir)/Makefile.private @@ -206,7 +213,7 @@ NETINSTALLABLE = core/pxelinux.0 gpxe/gpxelinux.0 core/lpxelinux.0 \ endif # ifdef EFI_BUILD -.PHONY: subdirs $(BSUBDIRS) $(ISUBDIRS) +.PHONY: subdirs $(BSUBDIRS) $(ISUBDIRS) test ifeq ($(HAVE_FIRMWARE),) -- cgit v1.2.3