diff --git a/ext/openssl/ossl.h b/ext/openssl/ossl.h index 7762ad237..0e59f64c4 100644 --- a/ext/openssl/ossl.h +++ b/ext/openssl/ossl.h @@ -121,6 +121,10 @@ rb_len2long(rb_len_t n) } #endif +#if !defined(FIX2INTPTR) +# define FIX2INTPTR FIX2LONG +#endif + #if !defined(NUM2UINT64T) /* in case Ruby starts to provide */ # if SIZEOF_LONG == 8 # define NUM2UINT64T(x) ((uint64_t)NUM2ULONG(x)) diff --git a/ext/openssl/ossl_bn.c b/ext/openssl/ossl_bn.c index ff5d3a538..00a1adc61 100644 --- a/ext/openssl/ossl_bn.c +++ b/ext/openssl/ossl_bn.c @@ -69,29 +69,23 @@ ossl_bn_new(const BIGNUM *bn) return obj; } -static BIGNUM * -integer_to_bnptr(VALUE obj, BIGNUM *orig) +static void +integer_to_bnptr(VALUE obj, BIGNUM *bn) { - BIGNUM *bn; - - if (FIXNUM_P(obj)) { - long i; - unsigned char bin[sizeof(long)]; - long n = FIX2LONG(obj); - unsigned long un = labs(n); - - for (i = sizeof(long) - 1; 0 <= i; i--) { - bin[i] = un & 0xff; - un >>= 8; - } + /* + * OpenSSL's BN_ULONG is usually pointer-sized, but is technically + * configurable and could be larger or smaller. + * LibreSSL's and AWS-LC's are always the same size as a pointer. + */ + if (sizeof(BN_ULONG) >= sizeof(uintptr_t) && FIXNUM_P(obj)) { + intptr_t n = FIX2INTPTR(obj); - bn = BN_bin2bn(bin, sizeof(bin), orig); - if (!bn) - ossl_raise(eBNError, "BN_bin2bn"); + if (!BN_set_word(bn, n < 0 ? -n : n)) + ossl_raise(eBNError, "BN_set_word"); if (n < 0) BN_set_negative(bn, 1); } - else { /* assuming Bignum */ + else { size_t len = rb_absint_size(obj, NULL); unsigned char *bin; VALUE buf; @@ -103,15 +97,13 @@ integer_to_bnptr(VALUE obj, BIGNUM *orig) bin = (unsigned char*)ALLOCV_N(unsigned char, buf, len); sign = rb_integer_pack(obj, bin, len, 1, 0, INTEGER_PACK_BIG_ENDIAN); - bn = BN_bin2bn(bin, (int)len, orig); + bn = BN_bin2bn(bin, (int)len, bn); ALLOCV_END(buf); if (!bn) ossl_raise(eBNError, "BN_bin2bn"); if (sign < 0) BN_set_negative(bn, 1); } - - return bn; } static VALUE @@ -124,8 +116,11 @@ try_convert_to_bn(VALUE obj) return obj; if (RB_INTEGER_TYPE_P(obj)) { newobj = NewBN(cBN); /* Handle potential mem leaks */ - bn = integer_to_bnptr(obj, NULL); + bn = BN_new(); + if (!bn) + ossl_raise(eBNError, "BN_new"); SetBN(newobj, bn); + integer_to_bnptr(obj, bn); } return newobj;