From d2cd45415dd6044247a1a2343f80bc072c40625b Mon Sep 17 00:00:00 2001 From: jrobinso Date: Wed, 8 Dec 2021 20:58:36 -0800 Subject: [PATCH] Add support for https requests over basic-auth proxy. Possible fix for #1062 --- .../java/org/broad/igv/util/HttpUtils.java | 77 +++-- .../igv/util/ProxiedHttpsConnection.java | 270 ++++++++++++++++++ .../org/broad/igv/prefs/preferences.tab | 2 +- test/server/noResponseServer.js | 24 ++ test/server/proxyServer.js | 80 ++++++ 5 files changed, 407 insertions(+), 46 deletions(-) create mode 100644 src/main/java/org/broad/igv/util/ProxiedHttpsConnection.java create mode 100644 test/server/noResponseServer.js create mode 100644 test/server/proxyServer.js diff --git a/src/main/java/org/broad/igv/util/HttpUtils.java b/src/main/java/org/broad/igv/util/HttpUtils.java index de1798a082..4589450605 100644 --- a/src/main/java/org/broad/igv/util/HttpUtils.java +++ b/src/main/java/org/broad/igv/util/HttpUtils.java @@ -26,8 +26,6 @@ package org.broad.igv.util; import biz.source_code.base64Coder.Base64Coder; -import htsjdk.samtools.seekablestream.SeekableStream; -import htsjdk.samtools.util.RuntimeIOException; import htsjdk.samtools.util.ftp.FTPClient; import htsjdk.samtools.util.ftp.FTPStream; import org.apache.log4j.Logger; @@ -711,56 +709,37 @@ private HttpURLConnection openConnection( url = addQueryParameter(url, "userProject", GoogleUtils.getProjectID()); } - Proxy sysProxy = null; - boolean igvProxySettingsExist = proxySettings != null && proxySettings.useProxy; - boolean checkSystemProxy = - !PreferencesManager.getPreferences().getAsBoolean("PROXY.DISABLE_CHECK") && !igvProxySettingsExist; + HttpURLConnection conn = null; + if (proxySettings != null && proxySettings.isProxyDefined()) { - //Only check for system proxy if igv proxy settings not found - if (checkSystemProxy) { - sysProxy = getSystemProxy(url.toExternalForm()); - } - - boolean useProxy = - (sysProxy != null && sysProxy.type() != Proxy.Type.DIRECT) || - (igvProxySettingsExist && !proxySettings.getWhitelist().contains(url.getHost())); - - HttpURLConnection conn; - if (useProxy) { - Proxy proxy = sysProxy; - if (igvProxySettingsExist) { - if (proxySettings.type == Proxy.Type.DIRECT) { + // NOTE: setting disabledSchemes to "" through System.setProperty does not work !!! Use ProxiedHttpsConnection + // System.setProperty("jdk.http.auth.tunneling.disabledSchemes", ""); + // System.setProperty("jdk.http.auth.proxying.disabledSchemes", ""); - if (PreferencesManager.getPreferences().getAsBoolean("DEBUG.PROXY")) { - log.info("NO_PROXY"); - } - - proxy = Proxy.NO_PROXY; - } else { - if (PreferencesManager.getPreferences().getAsBoolean("DEBUG.PROXY")) { - log.info("PROXY " + proxySettings.proxyHost + " " + proxySettings.proxyPort); - } - - proxy = new Proxy(proxySettings.type, new InetSocketAddress(proxySettings.proxyHost, proxySettings.proxyPort)); + if (url.getProtocol().equals("https") && proxySettings.isUserPwDefined()) { + conn = new ProxiedHttpsConnection(url, proxySettings.proxyHost, proxySettings.proxyPort, + proxySettings.user, proxySettings.pw); + } else { + Proxy proxy = new Proxy(proxySettings.type, new InetSocketAddress(proxySettings.proxyHost, proxySettings.proxyPort)); + conn = (HttpURLConnection) url.openConnection(proxy); + if (proxySettings.isUserPwDefined()) { + byte[] bytes = (proxySettings.user + ":" + proxySettings.pw).getBytes(); + String encodedUserPwd = String.valueOf(Base64Coder.encode(bytes)); + conn.setRequestProperty("Proxy-Authorization", "Basic " + encodedUserPwd); } } - conn = (HttpURLConnection) url.openConnection(proxy); - if (igvProxySettingsExist && proxySettings.auth && proxySettings.user != null && proxySettings.pw != null) { - byte[] bytes = (proxySettings.user + ":" + proxySettings.pw).getBytes(); - - String encodedUserPwd = String.valueOf(Base64Coder.encode(bytes)); - conn.setRequestProperty("Proxy-Authorization", "Basic " + encodedUserPwd); - } - } else { - if (PreferencesManager.getPreferences().getAsBoolean("DEBUG.PROXY")) { - log.info("PROXY NOT USED "); - if (proxySettings.getWhitelist().contains(url.getHost())) { - //log.info(url.getHost() + " is whitelisted"); - } + } + if (conn == null && !PreferencesManager.getPreferences().getAsBoolean("PROXY.DISABLE_CHECK")) { + Proxy sysProxy = getSystemProxy(url.toExternalForm()); + if (sysProxy != null && sysProxy.type() != Proxy.Type.DIRECT) { + conn = (HttpURLConnection) url.openConnection(sysProxy); } - conn = (HttpURLConnection) url.openConnection(); } + // If connection is still null no proxy is used + if (conn == null) { + conn = (HttpURLConnection) url.openConnection(); + } if (!"HEAD".equals(method)) { conn.setRequestProperty("Accept", "text/plain"); @@ -1049,6 +1028,14 @@ public ProxySettings(boolean useProxy, String user, String pw, boolean auth, Str this.whitelist = whitelist; } + public boolean isProxyDefined() { + return useProxy && proxyHost != null && proxyPort > 0; + } + + public boolean isUserPwDefined() { + return this.auth && this.user != null && this.pw != null; + } + public Set getWhitelist() { return whitelist; } diff --git a/src/main/java/org/broad/igv/util/ProxiedHttpsConnection.java b/src/main/java/org/broad/igv/util/ProxiedHttpsConnection.java new file mode 100644 index 0000000000..941039bddc --- /dev/null +++ b/src/main/java/org/broad/igv/util/ProxiedHttpsConnection.java @@ -0,0 +1,270 @@ +// URLConnection class for accessing https resources from a proxy authenticated with basic authentication +// This is nto supported in Java 11, and posted solutions to enable https with basic auth no longer work +// Class adapted from the Stack Overflow answer posted here: +// https://stackoverflow.com/a/35062001 + + +package org.broad.igv.util; + +import biz.source_code.base64Coder.Base64Coder; +import org.broad.igv.Globals; + +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.*; +import java.util.*; + +public class ProxiedHttpsConnection extends HttpURLConnection { + + private final String proxyHost; + private final int proxyPort; + private static final byte[] NEWLINE = "\r\n".getBytes();//should be "ASCII7" + + private Socket socket; + private FilteredInputStream inputStream; + private final Map> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + private final Map> sendheaders = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + private final Map> proxyheaders = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + private final Map> proxyreturnheaders = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + private int statusCode; + private String statusLine; + + public ProxiedHttpsConnection(URL url, String proxyHost, int proxyPort, String username, String password) + throws IOException { + super(url); + socket = new Socket(); + this.proxyHost = proxyHost; + this.proxyPort = proxyPort; + char[] encoded = Base64Coder.encode((username + ":" + password).getBytes()); + proxyheaders.put("Proxy-Authorization", new ArrayList<>(Arrays.asList("Basic " + encoded))); + } + + + @Override + public int getResponseCode() throws IOException { + connect(); + inputStream.parseHeaders(); + return statusCode; + } + + @Override + public InputStream getInputStream() throws IOException { + connect(); + return inputStream; + } + + @Override + public void setRequestMethod(String method) throws ProtocolException { + this.method = method; + } + + @Override + public void setRequestProperty(String key, String value) { + sendheaders.put(key, new ArrayList<>(Arrays.asList(value))); + } + + @Override + public void addRequestProperty(String key, String value) { + sendheaders.computeIfAbsent(key, l -> new ArrayList<>()).add(value); + } + + @Override + public Map> getHeaderFields() { + return headers; + } + + @Override + public void connect() throws IOException { + if (connected) { + return; + } + connected = true; + socket.setSoTimeout(getReadTimeout()); + socket.connect(new InetSocketAddress(proxyHost, proxyPort), getConnectTimeout()); + StringBuilder msg = new StringBuilder(); + msg.append("CONNECT "); + msg.append(url.getHost()); + msg.append(':'); + msg.append(url.getPort() == -1 ? 443 : url.getPort()); + msg.append(" HTTP/1.0\r\n"); + for (Map.Entry> header : proxyheaders.entrySet()) { + for (String l : header.getValue()) { + msg.append(header.getKey()).append(": ").append(l); + msg.append("\r\n"); + } + } + + msg.append("Connection: close\r\n"); + msg.append("\r\n"); + byte[] bytes; + try { + bytes = msg.toString().getBytes("ASCII7"); + } catch (UnsupportedEncodingException ignored) { + bytes = msg.toString().getBytes(); + } + socket.getOutputStream().write(bytes); + socket.getOutputStream().flush(); + + byte reply[] = new byte[200]; + byte header[] = new byte[200]; + int replyLen = 0; + int headerLen = 0; + int newlinesSeen = 0; + boolean headerDone = false; + /* Done on first newline */ + InputStream in = socket.getInputStream(); + while (newlinesSeen < 2) { + int i = in.read(); + if (i < 0) { + throw new IOException("Unexpected EOF from remote server"); + } + if (i == '\n') { + if (newlinesSeen != 0) { + String h = new String(header, 0, headerLen); + String[] split = h.split(": "); + if (split.length != 1) { + proxyreturnheaders.computeIfAbsent(split[0], l -> new ArrayList<>()).add(split[1]); + } + } + headerDone = true; + ++newlinesSeen; + headerLen = 0; + } else if (i != '\r') { + newlinesSeen = 0; + if (!headerDone && replyLen < reply.length) { + reply[replyLen++] = (byte) i; + } else if (headerLen < reply.length) { + header[headerLen++] = (byte) i; + } + } + } + + String replyStr; + try { + replyStr = new String(reply, 0, replyLen, "ASCII7"); + } catch (UnsupportedEncodingException ignored) { + replyStr = new String(reply, 0, replyLen); + } + + // Some proxies return http/1.1, some http/1.0 even we asked for 1.0 + if (!replyStr.startsWith("HTTP/1.0 200") && !replyStr.startsWith("HTTP/1.1 200")) { + throw new IOException("Unable to tunnel. Proxy returns \"" + replyStr + "\""); + } + + + SSLSocket s = (SSLSocket) ((SSLSocketFactory) SSLSocketFactory.getDefault()) + .createSocket(socket, url.getHost(), url.getPort(), true); + s.startHandshake(); + socket = s; + msg.setLength(0); + msg.append(method); + msg.append(" "); + msg.append(url.toExternalForm()); //.split(String.valueOf(url.getPort()), -2)[1]); + msg.append(" HTTP/1.0\r\n"); + for (Map.Entry> h : sendheaders.entrySet()) { + for (String l : h.getValue()) { + msg.append(h.getKey()).append(": ").append(l); + msg.append("\r\n"); + } + } + if (method.equals("POST") || method.equals("PUT")) { + msg.append("Transfer-Encoding: Chunked\r\n"); + } + msg.append("Host: ").append(url.getHost()).append("\r\n"); + msg.append("Connection: close\r\n"); + msg.append("\r\n"); + try { + bytes = msg.toString().getBytes("ASCII7"); + } catch (UnsupportedEncodingException ignored) { + bytes = msg.toString().getBytes(); + } + socket.getOutputStream().write(bytes); + socket.getOutputStream().flush(); + + this.inputStream = new FilteredInputStream(socket.getInputStream()); + } + + @Override + public void disconnect() { + try { + socket.close(); + if(this.inputStream != null) { + this.inputStream.close(); + } + } catch (IOException ex) { + //Logger.getLogger(ProxiedHttpsConnection.class.getName()).log(Level.SEVERE, null, ex); + } + } + + @Override + public boolean usingProxy() { + return true; + } + + class FilteredInputStream extends InputStream { + + static final int MAX_HEADER_SIZE = 1000; + InputStream wrappedStream; + boolean headersRead = false; + + public FilteredInputStream(InputStream wrappedStream) { + this.wrappedStream = wrappedStream; + } + + @Override + public int read() throws IOException { + + if (headersRead) { + return wrappedStream.read(); + } else { + parseHeaders(); + return wrappedStream.read(); + } + } + + @Override + public void close() throws IOException { + wrappedStream.close(); + super.close(); + } + + private void parseHeaders() throws IOException { + if (headersRead) { + return; + } else { + int newLineCount = 0; + int headerLen = 0; + byte[] header = new byte[MAX_HEADER_SIZE]; + while (newLineCount < 2) { + int i = wrappedStream.read(); + if (i == '\n') { + String h = new String(header, 0, headerLen); + if (statusLine == null) { + statusLine = h; + String[] parts = Globals.whitespacePattern.split(h); + statusCode = Integer.parseInt(parts[1]); + } else { + String[] split = h.split(": "); + if (split.length != 1) { + headers.computeIfAbsent(split[0], l -> new ArrayList<>()).add(split[1]); + } + } + headerLen = 0; + newLineCount++; + } else if (i == '\r') { + // skip + } else { + newLineCount = 0; + if (headerLen < MAX_HEADER_SIZE) { + header[headerLen++] = (byte) i; + } + } + } + headersRead = true; + } + } + } +} diff --git a/src/main/resources/org/broad/igv/prefs/preferences.tab b/src/main/resources/org/broad/igv/prefs/preferences.tab index ceff07036a..8be3f8c2dc 100644 --- a/src/main/resources/org/broad/igv/prefs/preferences.tab +++ b/src/main/resources/org/broad/igv/prefs/preferences.tab @@ -198,7 +198,7 @@ MASTER_RESOURCE_FILE_KEY Data registry url string https://data.broadinstitute.or PROVISIONING.URL OAuth provisioning URL string null --- -BLAT_URL Blat url String https://genome.ucsc.edu/cgi-bin/hgBlat?userSeq=$SEQUENCE&type=DNA&db=$DB&output=json +BLAT_URL Blat url String http://genome.ucsc.edu/cgi-bin/hgBlat?userSeq=$SEQUENCE&type=DNA&db=$DB&output=json --- ## JBrowse Circular View Integration diff --git a/test/server/noResponseServer.js b/test/server/noResponseServer.js new file mode 100644 index 0000000000..bba6b3dfdc --- /dev/null +++ b/test/server/noResponseServer.js @@ -0,0 +1,24 @@ +/** + * Simulates a server that does not return a response. + * + * To run + * node noResponseServer.js + * + * URL : http://localhost:9999 + * + * @type {module:http} + */ + +const http = require("http") + +const host = 'localhost' +const port = 9999 + +const server = http.createServer((req, res) => { + //res.writeHead(200) + // res.end("Response from server") +}) +server.listen(port, host, () => { + console.log(`Server is running on http://${host}:${port}`) +}) + diff --git a/test/server/proxyServer.js b/test/server/proxyServer.js new file mode 100644 index 0000000000..2126c93b1b --- /dev/null +++ b/test/server/proxyServer.js @@ -0,0 +1,80 @@ +// Proxy server from https://github.com/kasattejaswi/nodejs-proxy-server +// Copyright (c) 2021 Tejaswi Kasat + +// Import of net module +const net = require("net") +const server = net.createServer() + +server.on("connection", (clientToProxySocket) => { + + console.log("Client connected to proxy") + + clientToProxySocket.once("data", (data) => { + + console.log(`data -> ${data.toString()}`) + + let dataString = data.toString() + let isTLSConnection = dataString.indexOf("CONNECT") !== -1 + + let serverPort = 80 + let serverAddress + if (isTLSConnection) { + serverPort = 443 + serverAddress = dataString + .split("CONNECT")[1] + .split(" ")[1] + .split(":")[0] + } else { + serverAddress = dataString.split("Host: ")[1].split("\r\n")[0] + } + console.log(serverAddress) + + // Require a password + //if(dataString.indexOf("Proxy-Authorization") < 0) { + // clientToProxySocket.write("HTTP/1.1 407 Proxy requires authentication\r\n\r\n") + // return; + //} + + // Creating a connection from proxy to destination server + let proxyToServerSocket = net.createConnection( + { + host: serverAddress, + port: serverPort, + }, + () => { + console.log("Proxy to server set up") + } + ) + + + if (isTLSConnection) { + clientToProxySocket.write("HTTP/1.1 200 OK\r\n\r\n") + } else { + proxyToServerSocket.write(data) + } + + clientToProxySocket.pipe(proxyToServerSocket) + proxyToServerSocket.pipe(clientToProxySocket) + + + }) +}) + +server.on("error", (err) => { + console.log("Some internal server error occurred") + console.log(err) +}) + +server.on("close", () => { + console.log("Client disconnected") +}) + +server.listen( + { + host: "0.0.0.0", + port: 9999, + }, + () => { + console.log("Server listening on 0.0.0.0:9999") + } +) \ No newline at end of file