Fix pointer size issues on VMS

On VMS, the C compiler can work with 32-bit and 64-bit pointers, and
the command line determines what the initial pointer size shall be.

However, there is some functionality that only works with 32-bit
pointers.  In this case, it's gethostbyname(), getservbyname() and
accompanying structures, so we need to make sure that we define our
own pointers as 32-bit ones.

Furthermore, there seems to be a bug in VMS C netdb.h, where struct
addrinfo is always defined with 32-bit pointers no matter what, but
the functions handling it are adapted to the initial pointer size.
This leads to pointer size warnings when compiling with
/POINTER_SIZE=64.  The workaround is to force struct addrinfo to be
the 64-bit variant if the initial pointer size is 64.

Reviewed-by: Andy Polyakov <appro@openssl.org>

2 files changed, 49 insertions, 0 deletions

diff --git a/crypto/bio/b_addr.c b/crypto/bio/b_addr.c
index 663ec2ee07..ed4c1397d9 100644
--- a/crypto/bio/b_addr.c
+++ b/crypto/bio/b_addr.c
@@ -722,6 +722,15 @@ int BIO_lookup(const char *host, const char *service,
     } else {
 #endif
         const struct hostent *he;
+/*
+ * Because struct hostent is defined for 32-bit pointers only with
+ * VMS C, we need to make sure that '&he_fallback_address' and
+ * '&he_fallback_addresses' are 32-bit pointers
+ */
+#if defined(OPENSSL_SYS_VMS) && defined(__DECC)
+# pragma pointer_size save
+# pragma pointer_size 32
+#endif
         /* Windows doesn't seem to have in_addr_t */
 #ifdef OPENSSL_SYS_WINDOWS
         static uint32_t he_fallback_address;
@@ -735,6 +744,10 @@ int BIO_lookup(const char *host, const char *service,
         static const struct hostent he_fallback =
             { NULL, NULL, AF_INET, sizeof(he_fallback_address),
               (char **)&he_fallback_addresses };
+#if defined(OPENSSL_SYS_VMS) && defined(__DECC)
+# pragma pointer_size restore
+#endif
+
         struct servent *se;
         /* Apprently, on WIN64, s_proto and s_port have traded places... */
 #ifdef _WIN64
@@ -782,7 +795,19 @@ int BIO_lookup(const char *host, const char *service,
         } else {
             char *endp = NULL;
             long portnum = strtol(service, &endp, 10);
+
+/*
+ * Because struct servent is defined for 32-bit pointers only with
+ * VMS C, we need to make sure that 'proto' is a 32-bit pointer.
+ */
+#if defined(OPENSSL_SYS_VMS) && defined(__DECC)
+# pragma pointer_size save
+# pragma pointer_size 32
+#endif
             char *proto = NULL;
+#if defined(OPENSSL_SYS_VMS) && defined(__DECC)
+# pragma pointer_size restore
+#endif
 
             switch (socktype) {
             case SOCK_STREAM:
@@ -819,7 +844,19 @@ int BIO_lookup(const char *host, const char *service,
         *res = NULL;
 
         {
+/*
+ * Because hostent::h_addr_list is an array of 32-bit pointers with VMS C,
+ * we must make sure our iterator designates the same element type, hence
+ * the pointer size dance.
+ */
+#if defined(OPENSSL_SYS_VMS) && defined(__DECC)
+# pragma pointer_size save
+# pragma pointer_size 32
+#endif
             char **addrlistp;
+#if defined(OPENSSL_SYS_VMS) && defined(__DECC)
+# pragma pointer_size restore
+#endif
             size_t addresses;
             BIO_ADDRINFO *tmp_bai = NULL;
 
diff --git a/crypto/bio/bio_lcl.h b/crypto/bio/bio_lcl.h
index 1e409f8e13..7f3b22268a 100644
--- a/crypto/bio/bio_lcl.h
+++ b/crypto/bio/bio_lcl.h
@@ -32,6 +32,18 @@
 # endif
 
 # ifdef AI_PASSIVE
+
+/*
+ * There's a bug in VMS C header file netdb.h, where struct addrinfo
+ * always is the P32 variant, but the functions that handle that structure,
+ * such as getaddrinfo() and freeaddrinfo() adapt to the initial pointer
+ * size.  The easiest workaround is to force struct addrinfo to be the
+ * 64-bit variant when compiling in P64 mode.
+ */
+#  if defined(OPENSSL_SYS_VMS) && __INITIAL_POINTER_SIZE == 64
+#   define addrinfo __addrinfo64
+#  endif
+
 #  define bio_addrinfo_st addrinfo
 #  define bai_family      ai_family
 #  define bai_socktype    ai_socktype