aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorH. Peter Anvin (Intel) <hpa@zytor.com>2018-12-13 13:55:25 -0800
committerH. Peter Anvin (Intel) <hpa@zytor.com>2018-12-13 13:55:25 -0800
commit3b91f4c117003a9f42717fe88257b6025790169e (patch)
treee4e6af5c4151179113d084c99d480a794c1831aa
parent51222ab69e7ac1854587321442638620aa4829ba (diff)
downloadnasm-3b91f4c117003a9f42717fe88257b6025790169e.tar.gz
nasm-3b91f4c117003a9f42717fe88257b6025790169e.tar.xz
nasm-3b91f4c117003a9f42717fe88257b6025790169e.zip
malloc: handle potential infinite loop in nasm_alloc_failed()
It is possible on memory exhaustion that nasm_fatal() might cause another allocation error, thus calling nasm_alloc_failed() again. If we find us in nasm_alloc_failed() for a second time, try to get a message out and then call abort(). Signed-off-by: H. Peter Anvin (Intel) <hpa@zytor.com>
-rw-r--r--Makefile.in2
-rw-r--r--Mkfiles/msvc.mak2
-rw-r--r--Mkfiles/openwcom.mak2
-rw-r--r--asm/nasm.c6
-rw-r--r--include/error.h5
-rw-r--r--nasmlib/malloc.c18
-rw-r--r--rdoff/ldrdf.c3
7 files changed, 28 insertions, 10 deletions
diff --git a/Makefile.in b/Makefile.in
index 949d0131..32ef3d91 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -96,7 +96,7 @@ LIBOBJ = stdlib/snprintf.$(O) stdlib/vsnprintf.$(O) stdlib/strlcpy.$(O) \
stdlib/strnlen.$(O) stdlib/strrchrnul.$(O) \
\
nasmlib/ver.$(O) \
- nasmlib/crc64.$(O) nasmlib/malloc.$(O) \
+ nasmlib/crc64.$(O) nasmlib/malloc.$(O) nasmlib/errfile.$(O) \
nasmlib/md5c.$(O) nasmlib/string.$(O) \
nasmlib/file.$(O) nasmlib/mmap.$(O) nasmlib/ilog2.$(O) \
nasmlib/realpath.$(O) nasmlib/path.$(O) \
diff --git a/Mkfiles/msvc.mak b/Mkfiles/msvc.mak
index 9aec80fc..c7bd9109 100644
--- a/Mkfiles/msvc.mak
+++ b/Mkfiles/msvc.mak
@@ -68,7 +68,7 @@ LIBOBJ = stdlib\snprintf.$(O) stdlib\vsnprintf.$(O) stdlib\strlcpy.$(O) \
stdlib\strnlen.$(O) stdlib\strrchrnul.$(O) \
\
nasmlib\ver.$(O) \
- nasmlib\crc64.$(O) nasmlib\malloc.$(O) \
+ nasmlib\crc64.$(O) nasmlib\malloc.$(O) nasmlib\errfile.$(O) \
nasmlib\md5c.$(O) nasmlib\string.$(O) \
nasmlib\file.$(O) nasmlib\mmap.$(O) nasmlib\ilog2.$(O) \
nasmlib\realpath.$(O) nasmlib\path.$(O) \
diff --git a/Mkfiles/openwcom.mak b/Mkfiles/openwcom.mak
index 084014bb..11582ecf 100644
--- a/Mkfiles/openwcom.mak
+++ b/Mkfiles/openwcom.mak
@@ -57,7 +57,7 @@ LIBOBJ = stdlib\snprintf.$(O) stdlib\vsnprintf.$(O) stdlib\strlcpy.$(O) &
stdlib\strnlen.$(O) stdlib\strrchrnul.$(O) &
&
nasmlib\ver.$(O) &
- nasmlib\crc64.$(O) nasmlib\malloc.$(O) &
+ nasmlib\crc64.$(O) nasmlib\malloc.$(O) nasmlib\errfile.$(O) &
nasmlib\md5c.$(O) nasmlib\string.$(O) &
nasmlib\file.$(O) nasmlib\mmap.$(O) nasmlib\ilog2.$(O) &
nasmlib\realpath.$(O) nasmlib\path.$(O) &
diff --git a/asm/nasm.c b/asm/nasm.c
index ddc3404e..1825aa35 100644
--- a/asm/nasm.c
+++ b/asm/nasm.c
@@ -121,7 +121,7 @@ const struct ofmt *ofmt = &OF_DEFAULT;
const struct ofmt_alias *ofmt_alias = NULL;
const struct dfmt *dfmt;
-static FILE *error_file; /* Where to write error messages */
+FILE *error_file; /* Where to write error messages */
FILE *ofile = NULL;
struct optimization optimizing =
@@ -455,6 +455,8 @@ int main(int argc, char **argv)
timestamp();
+ error_file = stderr;
+
iflag_set_default_cpu(&cpu);
iflag_set_default_cpu(&cmd_cpu);
@@ -462,8 +464,6 @@ int main(int argc, char **argv)
want_usage = terminate_after_phase = false;
nasm_set_verror(nasm_verror_asm);
- error_file = stderr;
-
tolower_init();
src_init();
diff --git a/include/error.h b/include/error.h
index 5a676e67..477a26d7 100644
--- a/include/error.h
+++ b/include/error.h
@@ -41,6 +41,11 @@
#include "compiler.h"
/*
+ * File pointer for error messages
+ */
+extern FILE *error_file; /* Error file descriptor */
+
+/*
* An error reporting function should look like this.
*/
void printf_func(2, 3) nasm_error(int severity, const char *fmt, ...);
diff --git a/nasmlib/malloc.c b/nasmlib/malloc.c
index ccbc0c75..dbb7384a 100644
--- a/nasmlib/malloc.c
+++ b/nasmlib/malloc.c
@@ -44,7 +44,23 @@
static no_return nasm_alloc_failed(void)
{
- nasm_fatal(0, "out of memory");
+ /* If nasm_fatal() gets us back here, then croak hard */
+ static bool already_here = false;
+ FILE *errfile;
+
+ if (likely(!already_here)) {
+ already_here = true;
+ nasm_fatal(0, "out of memory!");
+ }
+
+ errfile = error_file;
+ if (!errfile)
+ error_file = stderr;
+
+ fprintf(error_file, "nasm: out of memory!\n");
+ fflush(error_file);
+ fflush(NULL);
+ abort();
}
static inline void *validate_ptr(void *p)
diff --git a/rdoff/ldrdf.c b/rdoff/ldrdf.c
index dd80d70e..49729b4f 100644
--- a/rdoff/ldrdf.c
+++ b/rdoff/ldrdf.c
@@ -126,9 +126,6 @@ char *generic_rec_file = NULL;
/* module name to be added at the beginning of output file */
char *modname_specified = NULL;
-/* error file */
-static FILE *error_file;
-
/* the header of the output file, built up stage by stage */
rdf_headerbuf *newheader = NULL;