Tests jsoup's client support for HTTPS

CHANGES

@@ -30,6 +30,10 @@ Release 1.17.1 [PENDING]
     DOMException. Now, said doctype is discarded, and the conversion continues.
 
   * Build Improvement: added a local test proxy implementation, for proxy integration tests.
+    <https://github.com/jhy/jsoup/pull/2029>
+
+  * Build Improvement: added tests for HTTPS request support, using a local self-signed cert. Includes proxy tests.
+    <https://github.com/jhy/jsoup/pull/2032>
 
 Release 1.16.2 [20-Oct-2023]
   * Improvement: optimized the performance of complex CSS selectors, by adding a cost-based query planner. Evaluators

pom.xml

@@ -405,6 +405,14 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <!-- jetty proxy, for integration tests -->
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-proxy</artifactId>
+      <version>${jetty.version}</version>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <!-- javax.annotations.nonnull, with Apache 2 (not GPL) license. Build time only. -->
       <groupId>com.google.code.findbugs</groupId>

src/test/java/org/jsoup/integration/ConnectTest.java

@@ -16,15 +16,19 @@
 import org.jsoup.parser.XmlTreeBuilder;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLDecoder;
+import java.nio.file.Files;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Stream;
 
 import static org.jsoup.helper.HttpConnection.CONTENT_TYPE;
 import static org.jsoup.helper.HttpConnection.MULTIPART_FORM_DATA;
@@ -51,6 +55,13 @@ public void canConnectToLocalServer() throws IOException {
         assertEquals("Hello, World!", p.text());
     }
 
+    @Test void canConnectToLocalTlsServer() throws IOException {
+        String url = HelloServlet.TlsUrl;
+        Document doc = Jsoup.connect(url).get();
+        Element p = doc.selectFirst("p");
+        assertEquals("Hello, World!", p.text());
+    }
+
     @Test
     public void fetchURl() throws IOException {
         Document doc = Jsoup.parse(new URL(echoUrl), 10 * 1000);
@@ -283,16 +294,16 @@ public void doesDeleteWithoutBody() throws IOException {
     /**
      * Tests upload of content to a remote service.
      */
-    @Test
-    public void postFiles() throws IOException {
+    @ParameterizedTest @MethodSource("echoUrls") // http and https
+    public void postFiles(String url) throws IOException {
         File thumb = ParseTest.getFile("/htmltests/thumb.jpg");
         File html = ParseTest.getFile("/htmltests/large.html");
 
         Document res = Jsoup
-            .connect(EchoServlet.Url)
+            .connect(url)
             .data("firstname", "Jay")
-            .data("firstPart", thumb.getName(), new FileInputStream(thumb), "image/jpeg")
-            .data("secondPart", html.getName(), new FileInputStream(html)) // defaults to "application-octetstream";
+            .data("firstPart", thumb.getName(), Files.newInputStream(thumb.toPath()), "image/jpeg")
+            .data("secondPart", html.getName(), Files.newInputStream(html.toPath())) // defaults to "application-octetstream";
             .data("surname", "Soup")
             .post();
 
@@ -786,4 +797,11 @@ public void maxBodySizeInReadToByteBuffer() throws IOException {
         assertEquals("%E9%8D%B5=%E5%80%A4", ihVal("Query String", doc));
         assertEquals("鍵=値", URLDecoder.decode(ihVal("Query String", doc), DataUtil.UTF_8.name()));
     }
+
+    /**
+     Provides HTTP and HTTPS EchoServlet URLs
+     */
+    private static Stream<String> echoUrls() {
+        return Stream.of(EchoServlet.Url, EchoServlet.TlsUrl);
+    }
 }

src/test/java/org/jsoup/integration/ProxyTest.java

@@ -11,15 +11,17 @@
 import org.jsoup.nodes.Element;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import java.io.IOException;
+import java.util.stream.Stream;
 
 import static org.jsoup.integration.ConnectTest.ihVal;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 /**
- Tests Jsoup.connect proxy support
- */
+ Tests Jsoup.connect proxy support */
 public class ProxyTest {
     private static String echoUrl;
     private static TestServer.ProxySettings proxy;
@@ -30,18 +32,23 @@ public static void setUp() {
         proxy = ProxyServlet.ProxySettings;
     }
 
-    @Test void fetchViaProxy() throws IOException {
-        Connection con = Jsoup.connect(HelloServlet.Url)
+    @ParameterizedTest @MethodSource("helloUrls")
+    void fetchViaProxy(String url) throws IOException {
+        Connection con = Jsoup.connect(url)
             .proxy(proxy.hostname, proxy.port);
 
         Connection.Response res = con.execute();
-        assertVia(res);
+        if (url.startsWith("http:/")) assertVia(res); // HTTPS CONNECT won't have Via
 
         Document doc = res.parse();
         Element p = doc.expectFirst("p");
         assertEquals("Hello, World!", p.text());
     }
 
+    private static Stream<String> helloUrls() {
+        return Stream.of(HelloServlet.Url, HelloServlet.TlsUrl);
+    }
+
     private static void assertVia(Connection.Response res) {
         assertEquals(res.header("Via"), ProxyServlet.Via);
     }
@@ -71,5 +78,11 @@ private static void assertVia(Connection.Response res) {
         assertVia(largeRes);
         assertEquals("Medium HTML", medRes.parse().title());
         assertEquals("Large HTML", largeRes.parse().title());
+
+        Connection.Response smedRes = session.newRequest().url(FileServlet.tlsUrlTo("/htmltests/medium.html")).execute();
+        Connection.Response slargeRes = session.newRequest().url(FileServlet.tlsUrlTo("/htmltests/large.html")).execute();
+
+        assertEquals("Medium HTML", smedRes.parse().title());
+        assertEquals("Large HTML", slargeRes.parse().title());
     }
 }

src/test/java/org/jsoup/integration/TestServer.java

@@ -1,75 +1,161 @@
 package org.jsoup.integration;
 
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.server.Connector;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.SecureRequestCustomizer;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.SslConnectionFactory;
+import org.eclipse.jetty.server.handler.HandlerWrapper;
 import org.eclipse.jetty.servlet.ServletHandler;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.jsoup.integration.servlets.BaseServlet;
 import org.jsoup.integration.servlets.ProxyServlet;
 
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
 
 public class TestServer {
-    private static final String localhost = "localhost";
-    private static final Server jetty = newServer();
-    private static final ServletHandler handler = new ServletHandler();
-    static int port;
+    static int Port;
+    static int TlsPort;
+
+    private static final String Localhost = "localhost";
+    private static final String KeystorePassword = "hunter2";
+
+    private static final Server Jetty = newServer();
+    private static final ServletHandler JettyHandler = new ServletHandler();
+    private static final Server Proxy = newServer();
+    private static final HandlerWrapper ProxyHandler = new HandlerWrapper();
+    private static final ProxySettings ProxySettings = new ProxySettings();
 
-    private static final Server proxy = newServer();
-    private static final ServletHandler proxyHandler = new ServletHandler();
-    private static final ProxySettings proxySettings = new ProxySettings();
 
     private static Server newServer() {
-        return new Server(new InetSocketAddress(localhost, 0));
+        return new Server(new InetSocketAddress(Localhost, 0));
     }
 
     static {
-        jetty.setHandler(handler);
-        proxy.setHandler(proxyHandler);
-        proxyHandler.addServletWithMapping(ProxyServlet.class, "/*");
+        Jetty.setHandler(JettyHandler);
+        Proxy.setHandler(ProxyHandler);
+
+        // TLS setup:
+        try {
+            File keystoreFile = ParseTest.getFile("/local-cert/server.pfx");
+            if (!keystoreFile.exists()) throw new FileNotFoundException(keystoreFile.toString());
+            addHttpsConnector(keystoreFile, Jetty);
+            setupDefaultTrust(keystoreFile);
+        } catch (Exception e) {
+            throw new IllegalStateException(e);
+        }
     }
 
     private TestServer() {
     }
 
     public static void start() {
-        synchronized (jetty) {
-            if (jetty.isStarted()) return;
+        synchronized (Jetty) {
+            if (Jetty.isStarted()) return;
 
             try {
-                jetty.start(); // jetty will safely no-op a start on an already running instance
-                port = ((ServerConnector) jetty.getConnectors()[0]).getLocalPort();
+                Jetty.start();
+                Connector[] jcons = Jetty.getConnectors();
+                Port = ((ServerConnector) jcons[0]).getLocalPort();
+                TlsPort = ((ServerConnector) jcons[1]).getLocalPort();
 
-                proxy.start();
-                proxySettings.port = ((ServerConnector) proxy.getConnectors()[0]).getLocalPort();
+                ProxyHandler.setHandler(ProxyServlet.createHandler());
+                Proxy.start();
+                ProxySettings.port = ((ServerConnector) Proxy.getConnectors()[0]).getLocalPort();
             } catch (Exception e) {
                 throw new IllegalStateException(e);
             }
         }
     }
 
-    public static String map(Class<? extends BaseServlet> servletClass) {
-        synchronized (jetty) {
-            if (!jetty.isStarted())
+    public static ServletUrls map(Class<? extends BaseServlet> servletClass) {
+        synchronized (Jetty) {
+            if (!Jetty.isStarted())
                 start(); // if running out of the test cases
 
             String path = "/" + servletClass.getSimpleName();
-            handler.addServletWithMapping(servletClass, path + "/*");
-            return "http://" + localhost + ":" + port + path;
+            JettyHandler.addServletWithMapping(servletClass, path + "/*");
+            String url = "http://" + Localhost + ":" + Port + path;
+            String tlsUrl = "https://" + Localhost + ":" + TlsPort + path;
+
+            return new ServletUrls(url, tlsUrl);
+        }
+    }
+
+    public static class ServletUrls {
+        public final String url;
+        public final String tlsUrl;
+
+        public ServletUrls(String url, String tlsUrl) {
+            this.url = url;
+            this.tlsUrl = tlsUrl;
         }
     }
 
     public static ProxySettings proxySettings() {
-        synchronized (jetty) {
-            if (!jetty.isStarted())
-                start(); // if running out of the test cases
+        synchronized (Jetty) {
+            if (!Jetty.isStarted())
+                start();
 
-            return proxySettings;
+            return ProxySettings;
         }
     }
 
     //public static String proxy
     public static class ProxySettings {
-        final String hostname = localhost;
+        final String hostname = Localhost;
         int port;
     }
+
+    private static void addHttpsConnector(File keystoreFile, Server server) {
+        // Cribbed from https://github.com/jetty/jetty.project/blob/jetty-9.4.x/examples/embedded/src/main/java/org/eclipse/jetty/embedded/LikeJettyXml.java
+        SslContextFactory sslContextFactory = new SslContextFactory.Server();
+        String path = keystoreFile.getAbsolutePath();
+        sslContextFactory.setKeyStorePath(path);
+        sslContextFactory.setKeyStorePassword(KeystorePassword);
+        sslContextFactory.setKeyManagerPassword(KeystorePassword);
+        sslContextFactory.setTrustStorePath(path);
+        sslContextFactory.setTrustStorePassword(KeystorePassword);
+
+        HttpConfiguration httpConfig = new HttpConfiguration();
+        httpConfig.setSecureScheme("https");
+        HttpConfiguration httpsConfig = new HttpConfiguration(httpConfig);
+        httpsConfig.addCustomizer(new SecureRequestCustomizer());
+
+        ServerConnector sslConnector = new ServerConnector(
+            server,
+            new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),
+            new HttpConnectionFactory(httpsConfig));
+        server.addConnector(sslConnector);
+    }
+
+    private static void setupDefaultTrust(File keystoreFile) throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException, KeyManagementException {
+        // Configure HttpsUrlConnection (jsoup) to trust (only) this cert
+        KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
+        trustStore.load(Files.newInputStream(keystoreFile.toPath()), KeystorePassword.toCharArray());
+        TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+        trustManagerFactory.init(trustStore);
+        TrustManager[] managers = trustManagerFactory.getTrustManagers();
+        SSLContext tls = SSLContext.getInstance("TLS");
+        tls.init(null, managers, null);
+        SSLSocketFactory socketFactory = tls.getSocketFactory();
+        HttpsURLConnection.setDefaultSSLSocketFactory(socketFactory);
+    }
 }

src/test/java/org/jsoup/integration/servlets/CookieServlet.java

@@ -9,7 +9,13 @@
 import java.io.PrintWriter;
 
 public class CookieServlet extends BaseServlet {
-    public static final String Url = TestServer.map(CookieServlet.class);
+    public static final String Url;
+    public static final String TlsUrl;
+    static {
+        TestServer.ServletUrls urls = TestServer.map(CookieServlet.class);
+        Url = urls.url;
+        TlsUrl = urls.tlsUrl;
+    }
     public static final String SetCookiesParam = "setCookies";
     public static final String LocationParam = "loc";
 

src/test/java/org/jsoup/integration/servlets/DeflateServlet.java

@@ -10,7 +10,13 @@
 import java.util.zip.DeflaterOutputStream;
 
 public class DeflateServlet extends BaseServlet {
-    public static final String Url = TestServer.map(DeflateServlet.class);
+    public static final String Url;
+    public static final String TlsUrl;
+    static {
+        TestServer.ServletUrls urls = TestServer.map(DeflateServlet.class);
+        Url = urls.url;
+        TlsUrl = urls.tlsUrl;
+    }
 
     @Override
     protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException {

src/test/java/org/jsoup/integration/servlets/EchoServlet.java

@@ -22,8 +22,14 @@
 
 public class EchoServlet extends BaseServlet {
     public static final String CodeParam = "code";
-    public static final String Url = TestServer.map(EchoServlet.class);
     private static final int DefaultCode = HttpServletResponse.SC_OK;
+    public static final String Url;
+    public static final String TlsUrl;
+    static {
+        TestServer.ServletUrls urls = TestServer.map(EchoServlet.class);
+        Url = urls.url;
+        TlsUrl = urls.tlsUrl;
+    }
 
     @Override
     protected void doIt(HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
@@ -114,7 +120,7 @@ static void write(PrintWriter w, String key, String val) {
     // allow the servlet to run as a main program, for local test
     public static void main(String[] args) {
         TestServer.start();
-        System.out.println(Url);
+        System.out.println("Listening on " + Url + " and " + TlsUrl);
     }
 
     private static boolean maybeEnableMultipart(HttpServletRequest req) {