From a53c292acb0914e5eb68aa77b06703f687358e32 Mon Sep 17 00:00:00 2001
From: Aditya Kamath <Aditya.Kamath1@ibm.com> 
Date: Mon, 15 Jun 2026 02:11:34 -0500
Subject: [PATCH] Add VAPI layer detection and adjust LR for frame backchain.

This is a patch which is used to detect a VAPI layer when raso llu_mode=1 is used in AIX 7.3 TL4 onwards.

When this is done then LR register is no longer suffficient to get the backtrace.
We need to look at the VAPI control block.

The original link register (LR) is saved in the VAPI control block (v_save_lr).
This happens in the VAPI glue code, which intercepts all calls into LLU-enabled libraries.

So the architectural reality is:

The LR in the stack/backchain is no longer sufficient to determine the true caller.
The actual return address is temporarily stored in the per-thread VAPI CB and restored on return.
---
 gdb/ppc-tdep.h        |   9 ++
 gdb/rs6000-aix-tdep.c | 211 ++++++++++++++++++++++++++++++++++++++++--
 gdb/rs6000-aix-vapi.h | 158 +++++++++++++++++++++++++++++++
 gdb/rs6000-tdep.c     |  38 ++++++++
 4 files changed, 407 insertions(+), 9 deletions(-)
 create mode 100644 gdb/rs6000-aix-vapi.h

diff --git a/gdb/ppc-tdep.h b/gdb/ppc-tdep.h
index 4dd20413125..aca8914f055 100644
--- a/gdb/ppc-tdep.h
+++ b/gdb/ppc-tdep.h
@@ -310,6 +310,15 @@ struct ppc_gdbarch_tdep : gdbarch_tdep_base
     struct type *ppc_builtin_type_vec128 = nullptr;
 
     int (*ppc_syscall_record) (struct regcache *regcache) = nullptr;
+
+    /* Platform-specific callback to adjust Link Register for trampolines.
+       Used by AIX to handle VAPI (Variable API) glue code that intercepts
+       library calls. When LR points to trampoline code, this callback reads
+       the real return address from platform-specific storage.
+       Returns true if LR was adjusted, false otherwise. */
+    bool (*ppc_adjust_trampoline_lr) (const frame_info_ptr &frame, CORE_ADDR pc,
+           CORE_ADDR *lr, int wordsize,
+           enum bfd_endian byte_order) = nullptr;
 };
 
 
diff --git a/gdb/rs6000-aix-tdep.c b/gdb/rs6000-aix-tdep.c
index d823fea5645..8c375a31e50 100644
--- a/gdb/rs6000-aix-tdep.c
+++ b/gdb/rs6000-aix-tdep.c
@@ -39,6 +39,9 @@
 #include "gdbsupport/xml-utils.h"
 #include "trad-frame.h"
 #include "frame-unwind.h"
+#include "rs6000-aix-vapi.h"
+#include <stdlib.h>
+#include <string.h>
 
 /* If the kernel has to deliver a signal, it pushes a sigcontext
    structure on the stack and then calls the signal handler, passing
@@ -66,7 +69,7 @@
 #define SIG_FRAME_FP_OFFSET64 168
 
 /* Minimum possible text address in AIX.  */
-#define AIX_TEXT_SEGMENT_BASE 0x10000000
+#define AIX_TEXT_SEGMENT_BASE 0x10000
 
 struct rs6000_aix_reg_vrreg_offset
 {
@@ -272,10 +275,18 @@ aix_sighandle_frame_cache (const frame_info_ptr &this_frame,
   else
     {
       func = read_memory_unsigned_integer (base_orig +
-					   SIG_FRAME_LR_OFFSET64,
-					   tdep->wordsize, byte_order);
-      safe_read_memory_integer (base_orig + SIG_FRAME_FP_OFFSET64,
-				tdep->wordsize, byte_order, &backchain);
+  			   SIG_FRAME_LR_OFFSET64,
+  			   tdep->wordsize, byte_order);
+      
+      /* Try reading backchain from offset 0 first, fallback to standard offset */
+      safe_read_memory_integer (base_orig, tdep->wordsize, byte_order, &backchain);
+      if (backchain == 0 || backchain == -1)
+ {
+   /* Fallback to standard offset if offset 0 doesn't work */
+   safe_read_memory_integer (base_orig + SIG_FRAME_FP_OFFSET64,
+  		    tdep->wordsize, byte_order, &backchain);
+ }
+      
       base = (CORE_ADDR)backchain;
     }
 
@@ -284,10 +295,72 @@ aix_sighandle_frame_cache (const frame_info_ptr &this_frame,
 
   if (tdep->wordsize == 4)
     trad_frame_set_reg_addr (this_trad_cache, tdep->ppc_lr_regnum,
-			     base_orig + 0x38 + 52 + 8);
+        base_orig + 0x38 + 52 + 8);
   else
     trad_frame_set_reg_addr (this_trad_cache, tdep->ppc_lr_regnum,
-			     base_orig + 0x70 + 320);
+        base_orig + 0x70 + 320);
+
+  /* Check if LR points to VAPI glue and recover from VAPI control block if needed.
+     This handles the case where signal handlers are called from VAPI-wrapped code.
+     VAPI glue code intercepts calls, storing real LR in TLS. */
+  {
+    CORE_ADDR lr_addr;
+    if (tdep->wordsize == 4)
+      lr_addr = base_orig + 0x38 + 52 + 8;
+    else
+      lr_addr = base_orig + 0x70 + 320;
+    
+    gdb_byte lr_buf[8];
+    if (target_read_memory (lr_addr, lr_buf, tdep->wordsize) == 0)
+      {
+ CORE_ADDR lr_value = extract_unsigned_integer (lr_buf, tdep->wordsize, byte_order);
+ 
+ /* Auto-detect VAPI: if LR points to VAPI glue code, read real LR from TLS */
+ CORE_ADDR pc_low = lr_value & 0xFFFF;
+ bool lr_in_vapi_glue = ((tdep->wordsize == 8 && pc_low >= vapi_addr_64 && pc_low < vapi_glue_addr_high) ||
+     (tdep->wordsize == 4 && pc_low >= vapi_glue_addr_low));
+ 
+ /* If LR points to VAPI glue, get real LR from VAPI TLS */
+ if (lr_in_vapi_glue)
+       {
+  try
+    {
+      CORE_ADDR tls_base = get_frame_register_unsigned (this_frame,
+      		      tdep->ppc_gp0_regnum + 13);
+      if (tls_base != 0)
+        {
+   CORE_ADDR vapi_lr_addr;
+   
+   if (tdep->wordsize == 8)
+     vapi_lr_addr = tls_base - VAPI_CB_OFFSET_64 + 8;
+   else
+     vapi_lr_addr = tls_base - VAPI_CB_OFFSET_32 + 8;
+   
+   gdb_byte vapi_buf[8];
+   if (target_read_memory (vapi_lr_addr, vapi_buf, tdep->wordsize) == 0)
+     {
+       CORE_ADDR vapi_lr_value = extract_unsigned_integer (vapi_buf,
+     	     tdep->wordsize,
+     	     byte_order);
+       
+       /* In LLU mode, use TLS value */
+       trad_frame_set_reg_value (this_trad_cache, tdep->ppc_lr_regnum,
+     vapi_lr_value);
+     }
+        }
+    }
+  catch (const gdb_exception_error &ex)
+    {
+      /* Ignore errors, use corrupted LR */
+    }
+       }
+     else
+       {
+  /* Normal LR or normal mode - use stack value */
+  trad_frame_set_reg_value (this_trad_cache, tdep->ppc_lr_regnum, lr_value);
+       }
+      }
+  }
 
   trad_frame_set_id (this_trad_cache, frame_id_build (base, func));
   trad_frame_set_this_base (this_trad_cache, base);
@@ -320,8 +393,37 @@ aix_sighandle_frame_sniffer (const struct frame_unwind *self,
 			     void **this_prologue_cache)
 {
   CORE_ADDR pc = get_frame_pc (this_frame);
-  if (pc && pc < AIX_TEXT_SEGMENT_BASE)
-    return 1;
+  
+  /* Signal handler frames on AIX have PC in a very low address range,
+     typically below AIX_TEXT_SEGMENT_BASE (0x10000).
+     
+     However, when VAPI is active, return addresses (like 0x8ec4
+     from at_quick_exit() in newer libc) can appear on the stack. These
+     should NOT be treated as signal handlers. Instead, the normal frame
+     unwinder should handle them and fetch the correct return address from
+     the VAPI control block in TLS.
+     
+     Therefore, we check if low PC is in VAPI glue range. If so, let the
+     VAPI-aware normal unwinder handle it. */
+  if (pc < AIX_TEXT_SEGMENT_BASE)
+    {
+      /* Check if PC is in VAPI glue range */
+      CORE_ADDR pc_low = pc & 0xFFFF;
+      struct gdbarch *gdbarch = get_frame_arch (this_frame);
+      ppc_gdbarch_tdep *tdep = gdbarch_tdep<ppc_gdbarch_tdep> (gdbarch);
+      bool is_vapi_glue = ((tdep->wordsize == 8 && pc_low >= vapi_addr_64 && pc_low < vapi_glue_addr_high) ||
+                           (tdep->wordsize == 4 && pc_low >= vapi_glue_addr_low));
+      
+      if (is_vapi_glue)
+ {
+   /* PC is in VAPI glue range, don't treat as signal handler.
+      Let the normal unwinder handle it with VAPI support. */
+   return 0;
+ }
+      
+      /* Not VAPI glue, treat as potential signal handler */
+      return 1;
+    }
 
   return 0;
 }
@@ -339,6 +441,91 @@ static const struct frame_unwind_legacy aix_sighandle_frame_unwind (
   aix_sighandle_frame_sniffer
 );
 
+/* AIX-specific callback for adjusting LR when it points to VAPI glue code.
+   VAPI (Variable API) is AIX's Live Library Update wrapper that intercepts
+   library calls. This function detects VAPI glue and reads the real return
+   address from the VAPI control block in TLS. */
+
+static bool
+rs6000_aix_adjust_vapi_lr (const frame_info_ptr &frame, CORE_ADDR pc,
+			    CORE_ADDR *lr, int wordsize,
+			    enum bfd_endian byte_order)
+{
+  /* Check if PC or LR is in VAPI glue code range */
+  CORE_ADDR pc_low = pc & 0xFFFF;
+  CORE_ADDR lr_low = *lr & 0xFFFF;
+  bool pc_in_vapi = false;
+  bool lr_in_vapi = false;
+  
+  if (wordsize == 8)  /* 64-bit */
+    {
+      pc_in_vapi = (pc_low >= vapi_addr_64 && pc_low < vapi_glue_addr_high);
+      lr_in_vapi = (lr_low >= vapi_addr_64 && lr_low < vapi_glue_addr_high);
+    }
+  else  /* 32-bit */
+    {
+      pc_in_vapi = (pc_low >= vapi_glue_addr_low);
+      lr_in_vapi = (lr_low >= vapi_glue_addr_low);
+    }
+  
+  /* Only adjust if PC or LR is in VAPI glue range */
+  if (!pc_in_vapi && !lr_in_vapi)
+    return false;
+  
+  if (pc_in_vapi)
+    {
+      gdb_printf (gdb_stdlog,
+		  "rs6000_in_vapi_glue_code: PC 0x%s (low=0x%x) detected as VAPI glue (%d-bit)\n",
+		  phex_nz (pc, wordsize),
+		  (unsigned int)pc_low,
+		  wordsize * 8);
+    }
+  
+  /* Try to get real LR from VAPI control block in TLS */
+  try
+    {
+      struct gdbarch *gdbarch = get_frame_arch (frame);
+      ppc_gdbarch_tdep *tdep = gdbarch_tdep<ppc_gdbarch_tdep> (gdbarch);
+      
+      CORE_ADDR tls_base = get_frame_register_unsigned (frame,
+							 tdep->ppc_gp0_regnum + 13);
+      
+      if (tls_base != 0)
+	{
+	  CORE_ADDR vapi_lr_addr;
+	  
+	  /* Calculate VAPI LR address using standard offset */
+	  if (wordsize == 8)
+	    vapi_lr_addr = tls_base - VAPI_CB_OFFSET_64 + 8;
+	  else
+	    vapi_lr_addr = tls_base - VAPI_CB_OFFSET_32 + 8;
+	  
+	  gdb_byte vapi_buf[8];
+	  if (target_read_memory (vapi_lr_addr, vapi_buf, wordsize) == 0)
+	    {
+	      CORE_ADDR vapi_lr_value = extract_unsigned_integer (vapi_buf,
+								   wordsize,
+								   byte_order);
+	      
+	      if (vapi_lr_value != 0 && vapi_lr_value != *lr)
+		{
+		  //gdb_printf (gdb_stdlog,
+		//	      "rs6000_frame_cache: Using VAPI v_save_lr 0x%s from TLS\n",
+		//	      phex_nz (vapi_lr_value, wordsize));
+		  *lr = vapi_lr_value;
+		  return true;
+		}
+	    }
+	}
+    }
+  catch (const gdb_exception_error &ex)
+    {
+      /* Ignore errors, don't adjust LR */
+    }
+  
+  return false;
+}
+
 /* Core file support.  */
 
 static struct ppc_reg_offsets rs6000_aix32_reg_offsets =
@@ -1410,7 +1597,13 @@ rs6000_aix_init_osabi (struct gdbarch_info info, struct gdbarch *gdbarch)
   set_gdbarch_wchar_signed (gdbarch, false);
   set_gdbarch_auto_wide_charset (gdbarch, rs6000_aix_auto_wide_charset);
 
+  /* Register AIX-specific callback for VAPI trampoline LR adjustment.
+     This allows the generic rs6000 frame unwinder to handle VAPI glue
+     code transparently without AIX-specific code in generic files. */
+  tdep->ppc_adjust_trampoline_lr = rs6000_aix_adjust_vapi_lr;
+
   set_gdbarch_make_solib_ops (gdbarch, make_aix_solib_ops);
+  
   frame_unwind_append_unwinder (gdbarch, &aix_sighandle_frame_unwind);
 }
 
diff --git a/gdb/rs6000-aix-vapi.h b/gdb/rs6000-aix-vapi.h
new file mode 100644
index 00000000000..f7e2477df20
--- /dev/null
+++ b/gdb/rs6000-aix-vapi.h
@@ -0,0 +1,158 @@
+/* AIX VAPI (Variable API) Layer Support for GDB
+   Copyright (C) 2026 Free Software Foundation, Inc.
+
+   This file is part of GDB.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+#ifndef RS6000_AIX_VAPI_H
+#define RS6000_AIX_VAPI_H
+
+/* ========================================================================
+   VAPI (Variable API) - AIX Live Library Update Support
+   ========================================================================
+   
+   VAPI is AIX's Live Library Update (LLU) wrapper layer. When llu_mode=1,
+   AIX intercepts library calls and wraps them with VAPI glue code, storing
+   the real return address in a Thread-Local Storage control block.
+   
+   Memory Layout (from r13 downward):
+   
+     r13 (TLS Base)
+       |
+       |--- TLS_POINTER_OFFSET (standard TLS data)
+       |
+       |--- VAPI_CB_SIZE (VAPI Control Block)
+       |      |
+       |      +--- Offset +0: v_save_pc (saved PC)
+       |      +--- Offset +8: v_save_lr (saved Link Register) ← WE NEED THIS!
+       |
+       |--- TLSCB_BASE_SIZE (TLS Control Block Base)
+       |
+       v (lower memory addresses)
+*/
+
+/* ========================================================================
+   VAPI Control Block Size Definitions
+   ======================================================================== */
+
+#define VAPI_CB_SIZE_32         128
+#define VAPI_CB_SIZE_64         256
+
+/* ========================================================================
+   Thread-Local Storage (TLS) Layout Definitions
+   ======================================================================== */
+
+#define TLS_POINTER_OFFSET32    (31 * 1024)  /* 31744 bytes */
+#define TLS_POINTER_OFFSET64    (30 * 1024)  /* 30720 bytes */
+
+#define TLSCB_BASE_SIZE32       128
+#define TLSCB_BASE_SIZE64       256
+
+/* ========================================================================
+   VAPI Control Block Offset Calculations
+   ======================================================================== */
+
+#define VAPI_CB_OFFSET_32 \
+	(TLS_POINTER_OFFSET32 + VAPI_CB_SIZE_32 + TLSCB_BASE_SIZE32)
+
+#define VAPI_CB_OFFSET_64 \
+	(TLS_POINTER_OFFSET64 + VAPI_CB_SIZE_64 + TLSCB_BASE_SIZE64)
+
+/* ========================================================================
+   VAPI Glue Code Address Ranges
+   ======================================================================== */
+
+#define vapi_glue_addr_ext_32   0x8B80
+#define vapi_addr_64            0x8E00
+#define vapi_size_64            0x0200
+
+/* User-friendly aliases */
+#define vapi_glue_addr_low      vapi_glue_addr_ext_32
+#define vapi_glue_addr_high     (vapi_addr_64 + vapi_size_64)
+
+/* ========================================================================
+   TLS Base Pointer Access Function (tptr)
+   ======================================================================== */
+
+#if !defined(__open_xl__)
+/* ========================================================================
+   VAPI Stack Frame Offsets
+   ======================================================================== */
+
+/* Offset from frame base where VAPI stores the backchain (saved SP)
+   When VAPI wraps a function, it stores the real backchain at this offset
+   within the stack frame (168 bytes from frame base) */
+#define VAPI_BACKCHAIN_OFFSET   0xa8
+
+/* For non-XL compilers, use pragma mc_func to define inline assembly */
+#ifdef __64BIT__
+char *tptr(void);
+#pragma mc_func tptr { "7da36b78"      /* mr r3,r13 */ }
+#pragma reg_killed_by tptr
+#else /* 32bit */
+char *tptr(void);
+#pragma mc_func tptr { "7C6342A6"      /* mfusprg3 r3 */ }
+#pragma reg_killed_by tptr
+#endif /* __64BIT__ */
+
+#else
+/* For XL compilers, use inline assembly with __asm__ */
+#ifdef __64BIT__
+static __inline__ __attribute__((__always_inline__)) char *tptr(void) {
+	register char *result __asm__("r3");
+	__asm__ volatile (
+		".long 0x7da36b78"  /* mr r3,r13 */
+		: "=r" (result)     /* output: result in r3 */
+		);
+	return result;
+}
+#else /* 32bit */
+static __inline__ __attribute__((__always_inline__)) char *tptr(void) {
+	register char *result __asm__("r3");
+	__asm__ volatile (
+		".long 0x7C6342A6"  /* mfusprg3 r3 */
+		: "=r" (result)     /* output: result in r3 */
+		);
+	return result;
+}
+#endif /* __64BIT__ */
+#endif
+
+/* ========================================================================
+   User-Facing Macros - Simple and Direct
+   ======================================================================== */
+
+/* Get the saved Link Register from VAPI control block (32-bit)
+   Usage: long *lr_ptr = (long *)GET_VAPI_LR_32;
+          long lr_value = *lr_ptr; */
+#define GET_VAPI_LR_32  (tptr() - VAPI_CB_OFFSET_32 + 8)
+
+/* Get the saved Link Register from VAPI control block (64-bit)
+   Usage: long *lr_ptr = (long *)GET_VAPI_LR_64;
+          long lr_value = *lr_ptr; */
+#define GET_VAPI_LR_64  (tptr() - VAPI_CB_OFFSET_64 + 8)
+
+/* Architecture-independent macro - automatically selects 32 or 64-bit
+   Usage: long *lr_ptr = (long *)GET_VAPI_LR;
+          printf("link register value:%lx\n", *lr_ptr); */
+#ifdef __64BIT__
+#define GET_VAPI_LR     GET_VAPI_LR_64
+#else
+#define GET_VAPI_LR     GET_VAPI_LR_32
+#endif
+
+#endif /* RS6000_AIX_VAPI_H */
+
+//
diff --git a/gdb/rs6000-tdep.c b/gdb/rs6000-tdep.c
index e135df94a40..9e638fd2742 100644
--- a/gdb/rs6000-tdep.c
+++ b/gdb/rs6000-tdep.c
@@ -3740,6 +3740,44 @@ rs6000_frame_cache (const frame_info_ptr &this_frame, void **this_cache)
   cache->saved_regs[gdbarch_pc_regnum (gdbarch)] =
     cache->saved_regs[tdep->ppc_lr_regnum];
 
+  /* Platform-specific: Check if LR points to trampoline code and adjust if needed.
+     This allows platforms like AIX to handle wrapper/glue code transparently. */
+  if (tdep->ppc_adjust_trampoline_lr != nullptr)
+    {
+      CORE_ADDR lr_value = 0;
+      
+      /* Try to get the LR value from saved_regs */
+      if (cache->saved_regs[tdep->ppc_lr_regnum].is_addr ())
+ {
+   CORE_ADDR lr_addr = cache->saved_regs[tdep->ppc_lr_regnum].addr ();
+   gdb_byte lr_buf[8];
+   if (target_read_memory (lr_addr, lr_buf, wordsize) == 0)
+     lr_value = extract_unsigned_integer (lr_buf, wordsize, byte_order);
+ }
+      else if (cache->saved_regs[tdep->ppc_lr_regnum].is_realreg ())
+ {
+   int lr_regnum = cache->saved_regs[tdep->ppc_lr_regnum].realreg ();
+   lr_value = get_frame_register_unsigned (this_frame, lr_regnum);
+ }
+      else if (cache->saved_regs[tdep->ppc_lr_regnum].is_value ())
+ {
+   lr_value = cache->saved_regs[tdep->ppc_lr_regnum].value ();
+ }
+      
+      /* Call platform-specific trampoline adjustment if LR was retrieved */
+      if (lr_value != 0)
+ {
+   CORE_ADDR adjusted_lr = lr_value;
+   if (tdep->ppc_adjust_trampoline_lr (this_frame, pc, &adjusted_lr,
+  			      wordsize, byte_order))
+     {
+       /* Trampoline detected and LR adjusted - update saved LR and PC */
+       cache->saved_regs[tdep->ppc_lr_regnum].set_value (adjusted_lr);
+       cache->saved_regs[gdbarch_pc_regnum (gdbarch)].set_value (adjusted_lr);
+     }
+ }
+    }
+
   /* If != 0, fdata.vrsave_offset is the offset from the frame that
      holds the VRSAVE.  */
   if (fdata.vrsave_offset != 0)
-- 
2.51.2

