_fseek_hdr(), _seek_hdr() does not recognize a.out with old-EMX startup code

[komh]

Hi/2.

Since kLIBC, _fseek_hdr() and _seek_hdr() has not recognized a.out with old-EMX startup code. So (pm)gdb cannot debug a.out with kLIBC startup code. In addition, the binaries made by Free Pascal which uses old-EMX startup code, are not supported by emxbind. See komh/cross-os2emx#30 for details.

Here are the testcases:
_fseek_hdr()
_seek_hdr()

Here is the patch: [Updated patch]

0001-_fseekhd-_seekhdr-Recognize-a.out-with-old-EMX-start.patch

From 8f212a20279494ab14454cd279f9c3fdfe8aa1f9 Mon Sep 17 00:00:00 2001
From: KO Myung-Hun <komh78@gmail.com>
Date: Sat, 16 May 2026 21:37:25 +0900
Subject: [PATCH] _fseekhd, _seekhdr: Recognize a.out with old-EMX startup code

Free Pascal uses a.out with old-EMX startup code.
---
 src/emx/src/lib/io/_fseekhd.c | 42 ++----------------------------
 src/emx/src/lib/io/_seekhdr.c | 48 ++++++++++++++++++++++++++++++++---
 2 files changed, 46 insertions(+), 44 deletions(-)

diff --git a/src/emx/src/lib/io/_fseekhd.c b/src/emx/src/lib/io/_fseekhd.c
index 2cec4fe..2f4c975 100644
--- a/src/emx/src/lib/io/_fseekhd.c
+++ b/src/emx/src/lib/io/_fseekhd.c
@@ -2,47 +2,9 @@
 
 #include "libc-alias.h"
 #include <stdio.h>
-#include <memory.h>
-#include <errno.h>
+#include <io.h>
 
 int _fseek_hdr (FILE *stream)
 {
-  struct
-    {
-      char magic[2];
-      char fill1[6];
-      unsigned short hdr_size;
-    } exe_hdr;
-  struct
-    {
-      char sig[16];
-      char bound;
-      char fill1;
-      unsigned short hdr_loc_lo;      /* cannot use long, alignment! */
-      unsigned short hdr_loc_hi;
-    } patch;
-  long original_pos;
-  int saved_errno;
-
-  original_pos = ftell (stream);
-  if (fread (&exe_hdr, sizeof (exe_hdr), 1, stream) != 1)
-    goto failure;
-  if (memcmp (exe_hdr.magic, "MZ", 2) != 0)
-    return (fseek (stream, original_pos, SEEK_SET) == -1 ? -1 : 0);
-  if (fseek (stream, original_pos + 16 * exe_hdr.hdr_size, SEEK_SET) == -1)
-    goto failure;
-  if (fread (&patch, sizeof (patch), 1, stream) != 1)
-    goto failure;
-  if (memcmp (patch.sig, "emx", 3) != 0)
-    goto failure;
-  if (fseek (stream, original_pos + patch.hdr_loc_lo
-             + 65536L * patch.hdr_loc_hi, SEEK_SET) == -1)
-    goto failure;
-  return 0;
-
-failure:
-  saved_errno = errno;
-  fseek (stream, original_pos, SEEK_SET);
-  errno = saved_errno;
-  return -1;
+  return _seek_hdr (fileno (stream));
 }
diff --git a/src/emx/src/lib/io/_seekhdr.c b/src/emx/src/lib/io/_seekhdr.c
index d4b2f57..e437833 100644
--- a/src/emx/src/lib/io/_seekhdr.c
+++ b/src/emx/src/lib/io/_seekhdr.c
@@ -33,10 +33,50 @@ int _seek_hdr (int handle)
     goto failure;
   if (read (handle, &patch, sizeof (patch)) != sizeof (patch))
     goto failure;
-  if (memcmp (patch.sig, "emx", 3) != 0)
-    goto failure;
-  if (lseek (handle, original_pos + patch.hdr_loc_lo
-              + 65536 * patch.hdr_loc_hi, SEEK_SET) == -1)
+  if (memcmp (patch.sig, "emx", 3) == 0)
+    {
+      if (lseek (handle, original_pos + patch.hdr_loc_lo
+                  + 65536 * patch.hdr_loc_hi, SEEK_SET) == -1)
+        goto failure;
+    }
+  else if (memcmp (patch.sig, "OS/2", 4) == 0)
+    {
+      struct
+        {
+          short fill1[12];
+          unsigned short lfarlc;
+          short fill2[17];
+          long lfanew;
+        } mz_hdr;
+      struct
+        {
+          char magic[2];
+          char fill1[2];
+          long fill2[31];
+          unsigned long data_page;
+        } lx_hdr;
+
+      /* rewind */
+      if (lseek (handle, original_pos, SEEK_SET) == -1)
+        goto failure;
+      if (read (handle, &mz_hdr, sizeof (mz_hdr)) != sizeof (mz_hdr))
+        goto failure;
+      /* check if a LX header exists */
+      if (mz_hdr.lfarlc != 0x40)
+        goto failure;
+      /* seek to the LX header */
+      if (lseek (handle, mz_hdr.lfanew, SEEK_SET) == -1)
+        goto failure;
+      if (read (handle, &lx_hdr, sizeof (lx_hdr)) != sizeof (lx_hdr))
+        goto failure;
+      /* check if real LX header */
+      if (memcmp (&lx_hdr.magic, "LX", 2) != 0)
+        goto failure;
+      /* seek to the aout header with ZMAGIC */
+      if (lseek (handle, lx_hdr.data_page - 1024, SEEK_SET) == -1)
+        goto failure;
+    }
+  else
     goto failure;
   return 0;
 
-- 
2.50.1