Because of some compile errors with openssl 1.0.0a, I’ve tried to change source code to match new openssl version and be able to recompile 1.8.7-p174 with new openssl version. Ruby 1.8.7-p299 doesn’t need any changes and works with openssl-1.0.0a like a charm.

I’m using rvm, so following steps will be described about that fact.

at first, you need to fetch $ rvm install ruby-1.8.7-p174 or go to the ruby-1.8.7 $ cd ~/.rvm/src/ruby-1.8.7-p174 . Then download/create the following patch and apply to the code patch -p1 < patch.txt . When patch is successfuly applied, try to run

$ rvm reload ruby-1.8.7-p174
$ rvm install ruby-1.8.7-p174

if everything is OK, you could continue with

$ rvm use 1.8.7-p174
$ irb
ruby-1.8.7-p174 > require 'openssl'
 => true 
ruby-1.8.7-p174 > OpenSSL::OPENSSL_VERSION
 => "OpenSSL 1.0.0a 1 Jun 2010"

patch is here:

diff -Naur ruby-1.8.7-p174_orig/ext/openssl/ossl.c ruby-1.8.7-p174/ext/openssl/ossl.c
--- ruby-1.8.7-p174_orig/ext/openssl/ossl.c	2007-06-08 17:02:04.000000000 +0200
+++ ruby-1.8.7-p174/ext/openssl/ossl.c	2010-07-02 23:08:11.000000000 +0200
@@ -15,7 +15,7 @@
  * String to HEXString conversion
  */
 int
-string2hex(char *buf, int buf_len, char **hexbuf, int *hexbuf_len)
+string2hex(const unsigned char *buf, int buf_len, char **hexbuf, int *hexbuf_len)
 {
     static const char hex[]="0123456789abcdef";
     int i, len = 2 * buf_len;
@@ -48,7 +48,7 @@
  * Data Conversion
  */
 STACK_OF(X509) *
-ossl_x509_ary2sk0(VALUE ary)  
+ossl_x509_ary2sk0(VALUE ary)
 {
     STACK_OF(X509) *sk;
     VALUE val;
@@ -57,13 +57,13 @@
 
     Check_Type(ary, T_ARRAY);
     sk = sk_X509_new_null();
-    if (!sk) ossl_raise(eOSSLError, NULL); 
+    if (!sk) ossl_raise(eOSSLError, NULL);
 
     for (i = 0; i < RARRAY_LEN(ary); i++) {
         val = rb_ary_entry(ary, i);
         if (!rb_obj_is_kind_of(val, cX509Cert)) {
             sk_X509_pop_free(sk, X509_free);
-            ossl_raise(eOSSLError, "object not X509 cert in array"); 
+            ossl_raise(eOSSLError, "object not X509 cert in array");
         }
         x509 = DupX509CertPtr(val); /* NEED TO DUP */
         sk_X509_push(sk, x509);
@@ -92,7 +92,7 @@
 
 #define OSSL_IMPL_SK2ARY(name, type)	        \
 VALUE						\
-ossl_##name##_sk2ary(STACK *sk)			\
+ossl_##name##_sk2ary(STACK_OF(type) *sk)	\
 {						\
     type *t;					\
     int i, num;					\
@@ -102,7 +102,7 @@
 	OSSL_Debug("empty sk!");		\
 	return Qnil;				\
     }						\
-    num = sk_num(sk);				\
+    num = sk_##type##_num(sk);			\
     if (num < 0) {				\
 	OSSL_Debug("items in sk < -1???");	\
 	return rb_ary_new();			\
@@ -110,7 +110,7 @@
     ary = rb_ary_new2(num);			\
 						\
     for (i=0; id.sign->cert;
-        crls = pkcs7->d.sign->crl;
         break;
     case NID_pkcs7_signedAndEnveloped:
         certs = pkcs7->d.signed_and_enveloped->cert;
-        crls = pkcs7->d.signed_and_enveloped->crl;
         break;
     default:
-        certs = crls = NULL;
+        certs = NULL;
     }
 
-    return want_certs ? certs : crls;
+    return certs;
+}
+static STACK_OF(X509_CRL) *
+pkcs7_get_crls(VALUE self)
+{
+    PKCS7 *pkcs7;
+    STACK_OF(X509_CRL) *crls;
+    int i;
+
+    GetPKCS7(self, pkcs7);
+    i = OBJ_obj2nid(pkcs7->type);
+    switch(i){
+      case NID_pkcs7_signed:
+        crls = pkcs7->d.sign->crl;
+        break;
+      case NID_pkcs7_signedAndEnveloped:
+        crls = pkcs7->d.signed_and_enveloped->crl;
+        break;
+      default:
+        crls = NULL;
+      }
+
+      return crls;
 }
 
 static VALUE
@@ -608,7 +627,7 @@
     STACK_OF(X509) *certs;
     X509 *cert;
 
-    certs = pkcs7_get_certs_or_crls(self, 1);
+    certs = pkcs7_get_certs(self);
     while((cert = sk_X509_pop(certs))) X509_free(cert);
     rb_block_call(ary, rb_intern("each"), 0, 0, ossl_pkcs7_set_certs_i, self);
 
@@ -618,7 +637,7 @@
 static VALUE
 ossl_pkcs7_get_certificates(VALUE self)
 {
-    return ossl_x509_sk2ary(pkcs7_get_certs_or_crls(self, 1));
+    return ossl_x509_sk2ary(pkcs7_get_certs(self));
 }
 
 static VALUE
@@ -648,7 +667,7 @@
     STACK_OF(X509_CRL) *crls;
     X509_CRL *crl;
 
-    crls = pkcs7_get_certs_or_crls(self, 0);
+    crls = pkcs7_get_crls(self);
     while((crl = sk_X509_CRL_pop(crls))) X509_CRL_free(crl);
     rb_block_call(ary, rb_intern("each"), 0, 0, ossl_pkcs7_set_crls_i, self);
 
@@ -658,7 +677,7 @@
 static VALUE
 ossl_pkcs7_get_crls(VALUE self)
 {
-    return ossl_x509crl_sk2ary(pkcs7_get_certs_or_crls(self, 0));
+    return ossl_x509crl_sk2ary(pkcs7_get_crls(self));
 }
 
 static VALUE
diff -Naur ruby-1.8.7-p174_orig/ext/openssl/ossl_ssl.c ruby-1.8.7-p174/ext/openssl/ossl_ssl.c
--- ruby-1.8.7-p174_orig/ext/openssl/ossl_ssl.c	2008-06-06 10:05:24.000000000 +0200
+++ ruby-1.8.7-p174/ext/openssl/ossl_ssl.c	2010-07-02 23:49:51.000000000 +0200
@@ -89,13 +89,19 @@
 static const char *ossl_ssl_attrs[] = { "sync_close", };
 
 ID ID_callback_state;
+#if OPENSSL_VERSION_NUMBER >= 0x10000000L
+#define OSSL_MORE_CONST const
+#define STACK _STACK
+#else
+#define OSSL_MORE_CONST
+#endif
 
 /*
  * SSLContext class
  */
 struct {
     const char *name;
-    SSL_METHOD *(*func)(void);
+    OSSL_MORE_CONST SSL_METHOD *(*func)(void);
 } ossl_ssl_method_tab[] = {
 #define OSSL_SSL_METHOD_ENTRY(name) { #name, name##_method }
     OSSL_SSL_METHOD_ENTRY(TLSv1),
@@ -122,8 +128,9 @@
 static void
 ossl_sslctx_free(SSL_CTX *ctx)
 {
+    if(ctx && ctx->references > 1) return;
     if(ctx && SSL_CTX_get_ex_data(ctx, ossl_ssl_ex_store_p)== (void*)1)
-	ctx->cert_store = NULL;
+	    ctx->cert_store = NULL;
     SSL_CTX_free(ctx);
 }
 
@@ -144,7 +151,7 @@
 static VALUE
 ossl_sslctx_set_ssl_version(VALUE self, VALUE ssl_method)
 {
-    SSL_METHOD *method = NULL;
+    OSSL_MORE_CONST SSL_METHOD *method = NULL;
     const char *s;
     int i;
 
@@ -585,7 +592,7 @@
 }
 
 static VALUE
-ossl_ssl_cipher_to_ary(SSL_CIPHER *cipher)
+ossl_ssl_cipher_to_ary(OSSL_MORE_CONST SSL_CIPHER *cipher)
 {
     VALUE ary;
     int bits, alg_bits;
@@ -609,7 +616,7 @@
 {
     SSL_CTX *ctx;
     STACK_OF(SSL_CIPHER) *ciphers;
-    SSL_CIPHER *cipher;
+    OSSL_MORE_CONST SSL_CIPHER *cipher;
     VALUE ary;
     int i, num;