Capture browser network logs running in Sauce labs - selenium-webdriver

Question

Capture browser network logs running in Saucelabs.
I used following code to catpuring browser network logs running in Saucelabs.
Its working in local machine, able to capture logs in local, but not able capture when running in saucelabs.
Here is the code. can anybody please though some light on this?
++++++++++ Local working fine +++++++++
package automation.networklogs;
import net.lightbody.bmp.BrowserMobProxy;
import net.lightbody.bmp.BrowserMobProxyServer;
import net.lightbody.bmp.client.ClientUtil;
import net.lightbody.bmp.core.har.Har;
import net.lightbody.bmp.proxy.CaptureType;
import org.openqa.selenium.By;
import org.openqa.selenium.MutableCapabilities;
import org.openqa.selenium.Proxy;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.remote.CapabilityType;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.openqa.selenium.remote.RemoteWebDriver;
import java.io.File;
import java.io.IOException;
import java.net.Inet4Address;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.HashMap;
public class NetWorkLogsWithMobProxyLocal {
public static String sFileName = "C:/Users/DELL/Desktop/New folder/harfile2.har";
public static WebDriver driver;
public static BrowserMobProxy proxy;
public static void main(String[] args) throws InterruptedException, MalformedURLException {
try {
proxy = new BrowserMobProxyServer();
proxy.start(0);
Proxy seleniumProxy = ClientUtil.createSeleniumProxy(proxy);
try {
String hostIp = Inet4Address.getLocalHost().getHostAddress();
seleniumProxy.setHttpProxy(hostIp + ":" + proxy.getPort());
seleniumProxy.setSslProxy(hostIp + ":" + proxy.getPort());
} catch (UnknownHostException e) {
e.printStackTrace();
}
DesiredCapabilities capabilities = new DesiredCapabilities();
capabilities.setCapability(CapabilityType.PROXY, seleniumProxy);
System.setProperty("webdriver.chrome.driver", "C:/Users/DELL/IdeaProjects/prodigy-client-repo/zenq/src/test/resources/browser_drivers/chromedriver.exe" );
ChromeOptions options = new ChromeOptions();
options.merge(capabilities);
driver = new ChromeDriver(options);
driver.manage().window().maximize();
proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);
proxy.newHar("yahoo.com" );
driver.get("https://yahoo.com" );
Thread.sleep(10);
Har har = proxy.getHar();
File harFile = new File(sFileName);
try {
har.writeTo(harFile);
} catch (IOException ex) {
System.out.println(ex.toString());
System.out.println("Could not find file " + sFileName);
}
} catch (Exception e) {
e.printStackTrace();
System.out.println("ENTER IN EXCEPTION firstName" );
} finally {
if (driver != null) {
proxy.stop();
driver.quit();
}
}
}
}
+++++++++ ++++++++++ saucelabs not working +++++++++
package automation.networklogs;
import net.lightbody.bmp.BrowserMobProxy;
import net.lightbody.bmp.BrowserMobProxyServer;
import net.lightbody.bmp.client.ClientUtil;
import net.lightbody.bmp.core.har.Har;
import net.lightbody.bmp.proxy.CaptureType;
import org.openqa.selenium.By;
import org.openqa.selenium.MutableCapabilities;
import org.openqa.selenium.Proxy;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.remote.CapabilityType;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.openqa.selenium.remote.RemoteWebDriver;
import java.io.File;
import java.io.IOException;
import java.net.Inet4Address;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.HashMap;
public class NetWorkLogsWithMobProxySL {
public static String sFileName = "C:/Users/DELL/Desktop/New folder/harfile3.har";
public static WebDriver driver;
public static BrowserMobProxy proxy;
public static void main(String[] args) throws InterruptedException, MalformedURLException {
try {
proxy = new BrowserMobProxyServer();
proxy.start(0);
Proxy seleniumProxy = ClientUtil.createSeleniumProxy(proxy);
String hostIp = Inet4Address.getLocalHost().getHostAddress();
seleniumProxy.setHttpProxy(hostIp + ":" + proxy.getPort());
seleniumProxy.setSslProxy(hostIp + ":" + proxy.getPort());
DesiredCapabilities capabilities = new DesiredCapabilities();
capabilities.setCapability(CapabilityType.PROXY, seleniumProxy);
MutableCapabilities sauceOpts = new MutableCapabilities();
sauceOpts.setCapability("username", "username" );
sauceOpts.setCapability("accessKey", "acceskey" );
sauceOpts.setCapability("seleniumVersion", "3.141.59" );
sauceOpts.setCapability("name", "test" );
sauceOpts.setCapability("proxy", hostIp+":"+proxy.getPort() );
HashMap<String, Object> chromePrefs = new HashMap<String, Object>();
chromePrefs.put("browser.visible", true);
ChromeOptions options = new ChromeOptions();
options.setExperimentalOption("prefs", chromePrefs);
options.merge(capabilities);
MutableCapabilities cap = new MutableCapabilities();
cap.setCapability("sauce:options", sauceOpts);
cap.setCapability("goog:chromeOptions", options);
cap.setCapability("platformName", "Windows 10" );
cap.setCapability("browserName", "chrome" );
cap.setCapability("browserVersion", "latest" );
String sauceURL = "https://ondemand.saucelabs.com:443/wd/hub";
driver = new RemoteWebDriver(new URL(sauceURL), cap);
proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);
proxy.newHar("yahoo.com" );
driver.get("https://yahoo.com" );
Thread.sleep(10);
Har har = proxy.getHar();
File harFile = new File(sFileName);
try {
har.writeTo(harFile);
} catch (IOException ex) {
System.out.println(ex.toString());
System.out.println("Could not find file " + sFileName);
}
} catch (Exception e) {
e.printStackTrace();
System.out.println("ENTER IN EXCEPTION firstName" );
} finally {
if (driver != null) {
proxy.stop();
driver.quit();
}
}
}
}
+++++++++ got below error for running in Saucelabs
log4j:WARN No appenders could be found for logger (io.netty.util.internal.logging.InternalLoggerFactory).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Sep 17, 2020 12:20:09 PM org.openqa.selenium.remote.ProtocolHandshake createSession
INFO: Detected dialect: W3C
org.openqa.selenium.WebDriverException: unknown error: net::ERR_PROXY_CONNECTION_FAILED
(Session info: chrome=85.0.4183.83)
Build info: version: '3.6.0', revision: '6fbf3ec767', time: '2017-09-27T15:28:36.4Z'
System info: host: 'RAVILAPTOP', ip: '192.168.147.1', os.name: 'Windows 10', os.arch: 'amd64', os.version: '10.0', java.version: '1.8.0_77'
Driver info: org.openqa.selenium.remote.RemoteWebDriver
Capabilities [{networkConnectionEnabled=false, chrome={chromedriverVersion=85.0.4183.38 (9047dbc2c693f044042bbec5c91401c708c7c26a-refs/branch-heads/4183#{#779}), userDataDir=C:\Users\ADMINI~1\AppData\Local\Temp\scoped_dir2300_1366152112}, timeouts={implicit=0, pageLoad=300000, script=30000}, pageLoadStrategy=normal, unhandledPromptBehavior=dismiss and notify, strictFileInteractability=false, platform=WINDOWS, proxy=Proxy(manual, http=192.168.147.1:59693, ssl=192.168.147.1:59693), goog:chromeOptions={debuggerAddress=localhost:49770}, browserVersion=85.0.4183.83, acceptInsecureCerts=false, browserName=chrome, javascriptEnabled=true, platformName=WINDOWS, setWindowRect=true, webauthn:virtualAuthenticators=true}]
Session ID: ebf16fbb5f7a4f5d8a91755b8a0966f9
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.openqa.selenium.remote.http.W3CHttpResponseCodec.createException(W3CHttpResponseCodec.java:185)
at org.openqa.selenium.remote.http.W3CHttpResponseCodec.decode(W3CHttpResponseCodec.java:120)
at org.openqa.selenium.remote.http.W3CHttpResponseCodec.decode(W3CHttpResponseCodec.java:49)
at org.openqa.selenium.remote.HttpCommandExecutor.execute(HttpCommandExecutor.java:164)
at org.openqa.selenium.remote.RemoteWebDriver.execute(RemoteWebDriver.java:586)
at org.openqa.selenium.remote.RemoteWebDriver.get(RemoteWebDriver.java:310)
at automation.networklogs.NetWorkLogsWithMobProxySL.main(NetWorkLogsWithMobProxySL.java:77)

Answer 1

You're getting org.openqa.selenium.WebDriverException: unknown error: net::ERR_PROXY_CONNECTION_FAILED because BrowserMob Proxy is running on your local machine, and is completely inaccessible from the Sauce Labs browser.
Normally, to access local-only resources during a test on Sauce Labs, you'd use Sauce Connect. That won't work here, because Sauce Connect is itself configured as a system proxy for the browser under test. If you configure a different proxy using Selenium's proxy settings, Sauce Connect will no longer work.
You can do one of two things. The complicated solution is to set up Sauce Connect, and configure it so that the client you run on your machine uses BrowserMob Proxy. There's some documentation on here.
The second, and much easier solution, is to use extended debugging. Then, Sauce Labs'll automatically capture the HAR file for you, along with JS Console logging. As long as the platform you're testing with supports it, you'll just have to add the desired capability extendedDebugging and set it to true. After a test is finished, you can download the HAR file from the REST API or Web UI.
(If you're using BrowserMob Proxy to alter requests as they come through, then you might find extended debugging's traffic editing useful... Although I personally recommend avoiding this kind of fiddling in tests as much as possible.)