OPSC-17995 Upgrade commons-compress 1.10 -> 1.27.1 to fix CVEs, bump version to 2.7.0.13

Jenkinsfile

@@ -0,0 +1,77 @@
+#!/usr/bin/env groovy
+
+@Library('ds-pipeline-lib')
+
+import java.security.MessageDigest
+def id = MessageDigest.getInstance("MD5").digest(System.currentTimeMillis().toString().bytes).encodeHex().toString().substring(0,8)
+
+// Build Jython forked repository and push artifacts to Artifactory
+// Addresses OPSC-17995: upgrades commons-compress 1.10 -> 1.27.1 to fix CVEs
+
+pipeline {
+    agent {
+        node {
+            label 'default-runner'
+            customWorkspace "workspace/${BUILD_NUMBER}-package-build-${id}"
+        }
+    }
+
+    tools {
+        jdk('jdk-8')
+    }
+
+    parameters {
+        string(name: 'buildbranch', defaultValue: 'ripcord-master', description: 'The branch to build.')
+    }
+
+    options {
+        disableConcurrentBuilds()
+        buildDiscarder(logRotator(numToKeepStr: '30'))
+        timeout(time: 120, unit: 'MINUTES')
+        timestamps()
+    }
+
+    stages {
+        stage('Build') {
+            steps {
+                script {
+                    env.JYTHON_VERSION = sh(
+                        script: "grep -m1 'property name=\"jython.version\"' build.xml | sed 's/.*value=\"\\([^\"]*\\)\".*/\\1/'",
+                        returnStdout: true
+                    ).trim()
+                    echo "Building jython-standalone version: ${env.JYTHON_VERSION}"
+                }
+                withAnt(installation: 'ant-1.10.7') {
+                    // Use pre-generated ANTLR sources (committed in gensrc/) to skip antlr_gen.
+                    // antlr-3.1.3 has a Java 8 NPE in ArrayList.removeAll; committing the
+                    // generated files means the ANTLR tool never needs to run in CI.
+                    sh "ant jar-standalone -Dgensrc.dir=${WORKSPACE}/gensrc"
+                    sh "mkdir -p artifacts"
+                    sh "cp dist/jython-standalone-*.jar artifacts/"
+                }
+                archiveArtifacts artifacts: 'dist/jython-standalone-*.jar', onlyIfSuccessful: true, defaultExcludes: false, caseSensitive: false
+            }
+        }
+        stage('Upload to Artifactory') {
+            steps {
+                script {
+                    def directoryPath = 'artifacts'
+                    def filenames = sh(script: "ls ${directoryPath}", returnStdout: true).trim().split('\n')
+
+                    withCredentials([usernamePassword(credentialsId: 'dse-artifactory',
+                                                      usernameVariable: 'ARTIFACTORY_USER',
+                                                      passwordVariable: 'ARTIFACTORY_PASSWORD')]) {
+                        for (def filename in filenames) {
+                            sh "curl -sSf -u '${ARTIFACTORY_USER}:${ARTIFACTORY_PASSWORD}' -X PUT -T artifacts/${filename} 'https://repo.aws.dsinternal.org/artifactory/datastax-public-releases-local/com/datastax/opscenter/jython-standalone/${env.JYTHON_VERSION}/${filename}'"
+                        }
+                    }
+                }
+            }
+        }
+        stage('Wrapup') {
+            steps {
+                cleanWs notFailBuild: true
+            }
+        }
+    }
+}

Lib/_sslcerts.py

@@ -10,15 +10,8 @@
 from java.security.cert import CertificateException, CertificateFactory
 from java.security.interfaces import RSAPrivateCrtKey
 from java.security.interfaces import RSAPublicKey
-from javax.net.ssl import X509KeyManager, X509TrustManager, KeyManagerFactory, SSLContext
-
-try:
-    # jarjar-ed version
-    from org.python.netty.handler.ssl.util import SimpleTrustManagerFactory
-
-except ImportError:
-    # dev version from extlibs
-    from io.netty.handler.ssl.util import SimpleTrustManagerFactory
+from javax.net.ssl import (
+    X509KeyManager, X509TrustManager, KeyManagerFactory, SSLContext, TrustManager, TrustManagerFactory)
 
 try:
     # dev version from extlibs OR if in classpath.
@@ -70,7 +63,7 @@ def _get_ca_certs_trust_manager(ca_certs=None):
             for cert in cf.generateCertificates(BufferedInputStream(f)):
                 trust_store.setCertificateEntry(str(uuid.uuid4()), cert)
                 num_certs_installed += 1
-    tmf = SimpleTrustManagerFactory.getInstance(SimpleTrustManagerFactory.getDefaultAlgorithm())
+    tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
     tmf.init(trust_store)
     log.debug("Installed %s certificates", num_certs_installed, extra={"sock": "*"})
     return tmf
@@ -335,7 +328,7 @@ def chooseServerAlias(self, key_type, issuers, socket):
 
     def getPrivateKey(self, alias):
         for key_manager in self.key_managers:
-            private_key = key_manager.getPrivateKey(alias)
+            private_key = keyManager.getPrivateKey(alias)
             if private_key:
                 return private_key
         return None
@@ -410,13 +403,14 @@ def getAcceptedIssuers(self):
         return certs
 
 
-class CompositeX509TrustManagerFactory(SimpleTrustManagerFactory):
+# To use with CERT_NONE
+class NoVerifyX509TrustManager(X509TrustManager):
 
-    def __init__(self, trust_managers):
-        self._trust_manager = CompositeX509TrustManager(trust_managers)
+    def checkClientTrusted(self, chain, auth_type):
+        pass
 
-    def engineInit(self, arg):
+    def checkServerTrusted(self, chain, auth_type):
         pass
 
-    def engineGetTrustManagers(self):
-        return [self._trust_manager]
+    def getAcceptedIssuers(self):
+        return None

Lib/ssl.py

@@ -6,6 +6,7 @@
 from java.io import BufferedInputStream, PrintWriter, StringWriter
 from java.security import KeyStore, KeyStoreException
 from java.security.cert import CertificateParsingException
+from javax.net.ssl import TrustManagerFactory
 from javax.naming.ldap import LdapName
 from java.lang import IllegalArgumentException, System
 import logging
@@ -18,16 +19,11 @@
 try:
     # jarjar-ed version
     from org.python.netty.channel import ChannelInitializer
-    from org.python.netty.handler.ssl import SslHandler, SslProvider, SslContextBuilder, ClientAuth
-    from org.python.netty.handler.ssl.util import SimpleTrustManagerFactory, InsecureTrustManagerFactory
-    from org.python.netty.buffer import ByteBufAllocator
-
+    from org.python.netty.handler.ssl import SslHandler
 except ImportError:
     # dev version from extlibs
     from io.netty.channel import ChannelInitializer
-    from io.netty.handler.ssl import SslHandler, SslProvider, SslContextBuilder, ClientAuth
-    from io.netty.handler.ssl.util import SimpleTrustManagerFactory, InsecureTrustManagerFactory
-    from io.netty.buffer import ByteBufAllocator
+    from io.netty.handler.ssl import SslHandler
 
 from _socket import (
     SSLError, raises_java_exception,
@@ -49,7 +45,7 @@
     error as socket_error)
 
 from _sslcerts import _get_openssl_key_manager, _extract_cert_from_data, _extract_certs_for_paths, \
-    _str_hash_key_entry, _get_ecdh_parameter_spec, CompositeX509TrustManagerFactory
+    NoVerifyX509TrustManager, _str_hash_key_entry, _get_ecdh_parameter_spec, CompositeX509TrustManager
 from _sslcerts import SSLContext as _JavaSSLContext
 
 from java.text import SimpleDateFormat
@@ -59,13 +55,6 @@
 from javax.security.auth.x500 import X500Principal
 from org.ietf.jgss import Oid
 
-try:
-    # requires Java 8 or higher for this support
-    from javax.net.ssl import SNIHostName, SNIMatcher
-    HAS_SNI = True
-except ImportError:
-    HAS_SNI = False
-
 log = logging.getLogger("_socket")
 ssl_log = logging.getLogger("jython_ssl")
 
@@ -77,10 +66,6 @@
 
 CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED = range(3)
 
-_CERT_TO_CLIENT_AUTH = {CERT_NONE: ClientAuth.NONE,
-                        CERT_OPTIONAL: ClientAuth.OPTIONAL,
-                        CERT_REQUIRED: ClientAuth.REQUIRE}
-
 # Do not support PROTOCOL_SSLv2, it is highly insecure and it is optional
 _, PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1, PROTOCOL_TLSv1_1, PROTOCOL_TLSv1_2 = range(6)
 _PROTOCOL_NAMES = {
@@ -104,8 +89,7 @@
 CHANNEL_BINDING_TYPES = []
 
 # https://docs.python.org/2/library/ssl.html#ssl.HAS_ALPN etc...
-HAS_ALPN, HAS_NPN, HAS_ECDH = False, False, True
-
+HAS_ALPN, HAS_NPN, HAS_ECDH, HAS_SNI = False, False, True, False
 
 # TODO not supported on jython yet
 # Disable weak or insecure ciphers by default
@@ -571,17 +555,12 @@ def wrap_child(child):
     def context(self):
         return self._context
 
-    @context.setter
-    def context(self, context):
-        self._context = context
-
     def setup_engine(self, addr):
         if self.engine is None:
             # http://stackoverflow.com/questions/13390964/java-ssl-fatal-error-80-unwrapping-net-record-after-adding-the-https-en
             self.engine = self._context._createSSLEngine(
                 addr, self.server_hostname,
-                cert_file=getattr(self, "certfile", None), key_file=getattr(self, "keyfile", None),
-                server_side=self.server_side)
+                cert_file=getattr(self, "certfile", None), key_file=getattr(self, "keyfile", None))
             self.engine.setUseClientMode(not self.server_side)
 
     def connect(self, addr):
@@ -1049,8 +1028,6 @@ def __init__(self, protocol, options=None):
 
         self._key_managers = None
 
-        self._server_name_callback = None
-
     def wrap_socket(self, sock, server_side=False,
                     do_handshake_on_connect=True,
                     suppress_ragged_eofs=True,
@@ -1061,30 +1038,24 @@ def wrap_socket(self, sock, server_side=False,
                          server_hostname=server_hostname,
                          _context=self)
 
-    def _createSSLEngine(self, addr, hostname=None, cert_file=None, key_file=None, server_side=False):
-        tmf = InsecureTrustManagerFactory.INSTANCE
-        if self.verify_mode != CERT_NONE:
-            # XXX need to refactor so we don't have to get trust managers twice
-            stmf = SimpleTrustManagerFactory.getInstance(SimpleTrustManagerFactory.getDefaultAlgorithm())
-            stmf.init(self._trust_store)
-
-            tmf = CompositeX509TrustManagerFactory(stmf.getTrustManagers())
+    def _createSSLEngine(self, addr, hostname=None, cert_file=None, key_file=None):
+        trust_managers = [NoVerifyX509TrustManager()]
+        if self.verify_mode == CERT_REQUIRED:
+            tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
             tmf.init(self._trust_store)
+            trust_managers = [CompositeX509TrustManager(tmf.getTrustManagers())]
 
-        kmf = self._key_managers
-        if self._key_managers is None:
-            kmf = _get_openssl_key_manager(cert_file=cert_file, key_file=key_file)
-
-        context_builder = None
-
-        if not server_side:
-            context_builder = SslContextBuilder.forClient()
+        context = _JavaSSLContext.getInstance(self._protocol_name)
 
-        if kmf:
-            if server_side:
-                context_builder = SslContextBuilder.forServer(kmf)
-            else:
-                context_builder = context_builder.keyManager(kmf)
+        if self._key_managers is None:
+            context.init(
+                _get_openssl_key_manager(
+                    cert_file=cert_file, key_file=key_file).getKeyManagers(),
+                trust_managers, None)
+        else:
+            context.init(
+                self._key_managers.getKeyManagers(),
+                trust_managers, None)
 
         # addr could be ipv6, only extract relevant parts
         engine = context.createSSLEngine((hostname or addr[0]), addr[1])
@@ -1093,22 +1064,14 @@ def _createSSLEngine(self, addr, hostname=None, cert_file=None, key_file=None, s
 
         engine.setSSLParameters(params)
 
-        context_builder = context_builder.trustManager(tmf)
-        context_builder = context_builder.sslProvider(SslProvider.JDK)
-        context_builder = context_builder.clientAuth(_CERT_TO_CLIENT_AUTH[self.verify_mode])
+        # apparently this can be used to enforce hostname verification
+        if hostname is not None and self._check_hostname:
+            params = engine.getSSLParameters()
+            params.setEndpointIdentificationAlgorithm('HTTPS')
+            engine.setSSLParameters(params)
 
         if self._ciphers is not None:
-            context_builder = context_builder.ciphers(self._ciphers)
-
-        if self._check_hostname:
-            engine = context_builder.build().newEngine(ByteBufAllocator.DEFAULT, hostname, addr[1])
-            if HAS_SNI:
-                params = engine.getSSLParameters()
-                params.setEndpointIdentificationAlgorithm('HTTPS')
-                params.setServerNames([SNIHostName(hostname)])
-                engine.setSSLParameters(params)
-        else:
-            engine = context_builder.build().newEngine(ByteBufAllocator.DEFAULT, addr[0], addr[1])
+            engine.setEnabledCipherSuites(self._ciphers)
 
         return engine
 
@@ -1192,10 +1155,7 @@ def set_npn_protocols(self, protocols):
         raise NotImplementedError()
 
     def set_servername_callback(self, server_name_callback):
-        if not callable(server_name_callback) and server_name_callback is not None:
-            raise TypeError("{!r} is not callable".format(server_name_callback))
-        self._server_name_callback = server_name_callback
-
+        raise NotImplementedError()
 
     def load_dh_params(self, dhfile):
         # TODO?

Lib/test/test_ssl.py

@@ -1521,7 +1521,7 @@ def test_algorithms(self):
         if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15):
             self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION)
         # sha256.tbs-internet.com needs SNI to use the correct certificate
-        if not ssl.HAS_SNI or support.is_jython:  # sha256.tbs-internet.com is no longer alive
+        if not ssl.HAS_SNI:
             self.skipTest("SNI needed for this test")
         # https://sha2.hboeck.de/ was used until 2011-01-08 (no route to host)
         remote = ("sha256.tbs-internet.com", 443)