Camel kafka with Spring FileStateRepository - apache-camel

I trying to save the Kafka offset into file i use Spring boot it seems the offset is writing in the file but not reading so the fact camel will start reading from the beginning of the kafka topic at restart
#Component
public class Route extends RouteBuilder {
#Override
public void configure() throws Exception {
from(kafka())
.to("log:TEST?level=INFO")
.process(Route::commitKafka);
}
private String kafka() {
String kafkaEndpoint = "kafka:";
kafkaEndpoint += "topic";
kafkaEndpoint += "?brokers=";
kafkaEndpoint += "localhost:9092";
kafkaEndpoint += "&groupId=";
kafkaEndpoint += "TEST";
kafkaEndpoint += "&autoOffsetReset=";
kafkaEndpoint += "earliest";
kafkaEndpoint += "&autoCommitEnable=";
kafkaEndpoint += false;
kafkaEndpoint += "&allowManualCommit=";
kafkaEndpoint += true;
kafkaEndpoint += "&offsetRepository=";
kafkaEndpoint += "#fileStore";
return kafkaEndpoint;
}
#Bean(name = "fileStore")
private FileStateRepository fileStateRepository() {
FileStateRepository fileStateRepository = FileStateRepository.fileStateRepository(new File("/kafka/offset_repo/repo.dat"));
// This will be empty
// System.out.println(fileStateRepository.getCache());
return fileStateRepository;
}
private static void commitKafka(Exchange exchange) {
KafkaManualCommit manual = exchange.getIn().getHeader(KafkaConstants.MANUAL_COMMIT, KafkaManualCommit.class);
manual.commitSync();
}
}

I finally found a solution but it does not appear in the documentation the start method must be called to init the repo at startup
#Bean(name = "fileStore")
private FileStateRepository fileStateRepository() {
FileStateRepository fileStateRepository = FileStateRepository.fileStateRepository(new File("/kafka/offset_repo/repo.dat"));
try {
fileStateRepository.start();
} catch (Exception e) {
e.printStackTrace();
}
return fileStateRepository;
}

Related

Extent report version 4 - Create two extent reports instead of one html report for all extectued testcases

I am using extent reportversion 4 and want one .html report after executing of all the testcases but it creates two html reports for executing the 3 methods in testclass
In the testclass, I have writteh code like that #beforemethod will execute before executing each testcase, followed by executing the testcase & in #aftermethod it will flush the repot to generate Html report and afterthat using #afterclass annotations to quit the driver**
**Testclass:**
public class HomePageTest extends BaseClass {
HomePage homePage;
public HomePageTest() {
super();
}
#BeforeMethod
#Parameters({ "platformName", "url", "udid" })
public void setUpHomePageClass(String platformName, String url, String udid) throws Exception {
try {
BaseClass baseClass = new BaseClass();
baseClass.initialize_driver(platformName, url, udid);
homePage = new HomePage(driver);
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
#BeforeMethod
#Parameters({ "platformName", "url", "udid" })
public void setUpHomePageClass(String platformName, String url, String udid) throws Exception {
try {
BaseClass baseClass = new BaseClass();
baseClass.initialize_driver(platformName, url, udid);
homePage = new HomePage(driver);
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
#Test(priority = 1, description = "Verify element i.e Top50 Txt on homepage test")
#Severity(SeverityLevel.NORMAL)
#Description("TestCase Description: Verify element i.e Top50 Txt on homepage")
public void verifyeElementsOnHomePageTest() throws Exception {
log.info("***Executing verifyElementsOnHomeScreenTest***");
logger = extent.createTest("Verify the elements on HomePage after redirecting to the splash screen");
log.info("wait for continue_button to be clickable");
TestUtil.waitForElementToBeClickable(By.id("continue_button"));
homePage.clickContinueBtnAfterSplashScreen();
log.info("Clicked on continue_button");
log.info("waitForUserNameToBeClickable - username");
boolean flag = homePage.validateTop50Txt();
Assert.assertTrue(flag);
log.info("Top50Txt isDisplayed");
log.info("verifyElementsonHomeScreenTest Ended");
}
#Test(priority = 2, description = "Swipe to next video test")
#Severity(SeverityLevel.NORMAL)
#Description("TestCase Description: Swipe from one video to another")
public void swipeToNxtVideoTest() throws InterruptedException {
try {
logger = extent.createTest("Swipe from one video to another & get the username ");
log.info("***Executing swipeToNxtVideoTest***");
log.info("waitForElementToPresenceOfElementLocated - username");
TestUtil.waitForElementToPresenceOfElementLocated(By.id("user_name"));
log.info("swipeverticalDown for nxt video");
TestUtil.swipeverticalDown();
log.info("swipeToNxtVideoTest Ended");
} catch (Exception e) {
e.printStackTrace();
log.error("Found Exception - swipeToNxtVideoTest");
}
}
/*
* #Test(priority = 3, retryAnalyzer =
* com.automation.listeners.RetryAnalyzer.class ) public void checkFailure() {
* Assert.assertEquals(true, false); System.out.println("failed");
*
* }
*/
#AfterMethod
public void getResult(ITestResult result) throws Exception {
if (result.getStatus() == ITestResult.FAILURE) {
logger.log(Status.FAIL,
MarkupHelper.createLabel(result.getName() + " - Test Case Failed", ExtentColor.RED));
logger.log(Status.FAIL,
MarkupHelper.createLabel(result.getThrowable() + " - Test Case Failed", ExtentColor.RED));
String screenshotPath = TestUtil.captureScreenAsBase64(driver, result.getName());
logger.fail("Snapshot below: " + logger.addScreenCaptureFromPath(screenshotPath));
} else if (result.getStatus() == ITestResult.SKIP) {
logger.log(Status.SKIP,
MarkupHelper.createLabel(result.getName() + " - Test Case Skipped", ExtentColor.ORANGE));
} else if (result.getStatus() == ITestResult.SUCCESS) {
logger.log(Status.PASS,
MarkupHelper.createLabel(result.getName() + " Test Case PASSED", ExtentColor.GREEN));
}
extent.flush();
}
#AfterClass
public void quitDriver() {
getDriver().quit();
}
Please do let me know where I have been lacking in code; I might have a intuitions that there is an issue in testng annotations in my code
Base Class:
DesiredCapabilities capabilities = new DesiredCapabilities();
public void setDriver(AppiumDriver<MobileElement> driver) {
tdriver.set(driver);
}
public static synchronized AppiumDriver<MobileElement> getDriver() {
return tdriver.get();
}
public BaseClass() {
try {
prop = new Properties();
FileInputStream ip = new FileInputStream(
System.getProperty("user.dir") + "/src/main/java/com/automation/config/config.properties");
prop.load(ip);
// extend reports
Date date = new Date();
SimpleDateFormat dateFormatFolder = new SimpleDateFormat("dd_MMM_yyyy");
File ResultDir = new File(System.getProperty("user.dir") + File.separator + "/FrameworkReports/"
+ dateFormatFolder.format(date));
// Defining Directory/Folder Name
if (!ResultDir.exists()) { // Checks that Directory/Folder Doesn't Exists!
ResultDir.mkdir();
}
SimpleDateFormat dateFormat = new SimpleDateFormat("dd_MMM_yyyy_hh_mm_ssaa");
htmlReporter = new ExtentHtmlReporter(
ResultDir + "/" + "Report" + " " + dateFormat.format(date) + " .html");
htmlReporter.config().setDocumentTitle("Automation Report");
htmlReporter.config().setReportName("YOVO AUTOMATION");
htmlReporter.config().setTheme(Theme.DARK);
extent = new ExtentReports();
extent.attachReporter(htmlReporter);
extent.setSystemInfo("Host Name", "localhost");
extent.setSystemInfo("Environment", "Windows 7");
extent.setSystemInfo("User Name", "Abhishek Chauhan");
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
public void initialize_driver(String platformName, String url, String udid) throws Exception {
log = LogManager.getLogger(BaseClass.class);
BasicConfigurator.configure();
File appDir = new File("/src/main/resources/apk");
File app = new File(appDir, "yovoapp-release.apk");
mDirpath = System.getProperty("user.dir");
mApkfilepath = mDirpath + "/app/yovoapp-release.apk";
capabilities.setCapability(MobileCapabilityType.PLATFORM_NAME, platformName);
capabilities.setCapability(MobileCapabilityType.UDID, udid);
switch (platformName) {
case "Android":
capabilities.setCapability(MobileCapabilityType.NEW_COMMAND_TIMEOUT, 60);
capabilities.setCapability("appPackage", prop.getProperty("androidAppPackage"));
capabilities.setCapability("appActivity", prop.getProperty("androidAppActivity"));
capabilities.setCapability("app", mApkfilepath);
capabilities.setCapability("noReset", true);
driver = new AppiumDriver<MobileElement>(new URL(url), capabilities);
// tdriver.set(driver);
// return getDriver();
case "IOS":
File classpathRoot = new File(System.getProperty("user.dir"));
// File appDir = new File(classpathRoot, "/build/");
// File app = new File(appDir, "WordPress.app");
capabilities.setCapability("platformVersion", "9.2");
capabilities.setCapability("deviceName", "iPhone 6");
capabilities.setCapability("app", app.getAbsolutePath());
// driver = new IOSDriver<MobileElement>(new
// URL("http://127.0.0.1:4723/wd/hub"), caps);
break;
default:
throw new Exception("Invalid platform! - " + platformName);
}
setDriver(driver);
}

Reading and writing the file from and to winSCP from the S3 object store

I am trying to put and read file from the remote file system using winSCP through an SFTP connection. The leaf node of the file system is s3 object store which contain the files (for eg: xyz.txt).
Below is the overridden method of File Channel class.
XYZFileSystemProvider
public class XYZFileSystemProvider extends FileSystemProvider {
#Override
public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs)
throws IOException {
// TODO Auto-generated method stub
Collection<XYZOptions.OpenMode> modes = XYZOptions.OpenMode.fromOpenOptions(options);
if (modes.isEmpty()) {
modes = EnumSet.of(XYZOptions.OpenMode.Read, MFEOptions.OpenMode.Write);
}
// TODO: process file attributes
return new XYZFileSystemChannel(path, modes);
}
}
XYZFileSystemChannel
public class XYZFileSystemChannel extends XYZRemotePathChannel{
public XYZFileSystemChannel(XYZPath p, Collection<XYZOptions.OpenMode> modes) throws IOException {
this(Objects.requireNonNull(p, "No target path").toString(), p.getFileSystem(), modes);
}
public XYZFileSystemChannel(String remotePath, XYZFileSystem fs, Collection<XYZOptions.OpenMode> modes) throws IOException {
super(remotePath, fs, true, modes);
}
}
XYZRemotePathChannel
public class XYZRemotePathChannel extends FileChannel {
private AmazonS3Component getAmazonS3Instance() {
return SpringContext.getBean(AmazonS3Component.class);
}
private final String path;
private final Collection<XYZOptions.OpenMode> modes;
private final boolean closeOnExit;
private XYZFileSystem fileSystem;
private final AtomicLong posTracker = new AtomicLong(0L);
public static final Set<XYZOptions.OpenMode> READ_MODES =
Collections.unmodifiableSet(EnumSet.of(XYZOptions.OpenMode.Read));
private final Object lock = new Object();
private final AtomicReference<Thread> blockingThreadHolder = new AtomicReference<>(null);
public XYZRemotePathChannel(String path, XYZFileSystem fileSystem, boolean closeOnExit,
Collection<XYZOptions.OpenMode> modes) throws IOException {
this.path = ValidateUtils.checkNotNullAndNotEmpty(path, "No remote file path specified");
this.modes = Objects.requireNonNull(modes, "No channel modes specified");
this.closeOnExit = closeOnExit;
this.fileSystem = fileSystem;
}
#Override
public int read(ByteBuffer dst) throws IOException {
// TODO Auto-generated method stub
log.debug("Position of dst is : {}",dst.position());
log.debug("Reading the bytes of the file : {}", dst);
//Some code to be done here in order to read dst and send bytes of the file recieved from s3 store
return (int) doRead(Collections.singletonList(dst), -1);
}
protected long doRead(List<ByteBuffer> buffers, long position) throws IOException {
log.debug("Do Reading the bytes of the file of list of buffer : {} and position :{}", buffers , position);
ensureOpen(READ_MODES);
synchronized (lock) {
boolean completed = false;
boolean eof = false;
long curPos = (position >= 0L) ? position : posTracker.get();
byte[] bytes = new byte[(int) curPos];
try {
long totalRead = 0;
beginBlocking();
String [] parts = this.path.toString().replaceFirst("^/", "").split("/");
String bucket = parts[parts.length-2];
String fileName = parts[parts.length-1];
InputStream fileContent = getAmazonS3Instance().getFileFromBucket(bucket, fileName);
log.debug("Contens of the file: {} from bucket: {} are : {}", fileName , bucket, fileContent);
//Some code to be done here to return the content byte length??
int fileLenght = fileContent.read(bytes, 1, (int) curPos);
log.debug("After reading the file content the file length is : {}" , fileLenght );
return fileLenght;
} finally {
if (position < 0L) {
posTracker.set(curPos);
}
endBlocking(completed);
}
}
}
private void endBlocking(boolean completed) throws AsynchronousCloseException {
blockingThreadHolder.set(null);
end(completed);
}
private void beginBlocking() {
begin();
blockingThreadHolder.set(Thread.currentThread());
}
#Override
public FileChannel position(long newPosition) throws IOException {
// TODO Auto-generated method stub
log.debug("Setting the position of the file : {}", newPosition);
if (newPosition < 0L) {
throw new IllegalArgumentException("position(" + this.path + ") illegal file channel position: " + newPosition);
}
ensureOpen(Collections.emptySet());
posTracker.set(newPosition);
return this;
}
private void ensureOpen(Collection<XYZOptions.OpenMode> reqModes) throws IOException {
if (!isOpen()) {
throw new ClosedChannelException();
}
if (GenericUtils.size(reqModes) > 0) {
for (XYZOptions.OpenMode m : reqModes) {
if (this.modes.contains(m)) {
return;
}
}
throw new IOException("ensureOpen(" + this.path + ") current channel modes (" + this.modes
+ ") do contain any of the required: " + reqModes);
}
}
}
XYZOptions
public class XYZOptions {
enum OpenMode {
Read, Write, Append, Create, Truncate, Exclusive;
public static final Set<OpenOption> SUPPORTED_OPTIONS = Collections
.unmodifiableSet(EnumSet.of(StandardOpenOption.READ, StandardOpenOption.APPEND,
StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE,
StandardOpenOption.CREATE_NEW, StandardOpenOption.SPARSE));
public static Set<OpenMode> fromOpenOptions(Collection<? extends OpenOption> options) {
if (GenericUtils.isEmpty(options)) {
return Collections.emptySet();
}
Set<OpenMode> modes = EnumSet.noneOf(OpenMode.class);
for (OpenOption option : options) {
if (option == StandardOpenOption.READ) {
modes.add(Read);
} else if (option == StandardOpenOption.APPEND) {
modes.add(Append);
} else if (option == StandardOpenOption.CREATE) {
modes.add(Create);
} else if (option == StandardOpenOption.TRUNCATE_EXISTING) {
modes.add(Truncate);
} else if (option == StandardOpenOption.WRITE) {
modes.add(Write);
} else if (option == StandardOpenOption.CREATE_NEW) {
modes.add(Create);
modes.add(Exclusive);
} else if (option == StandardOpenOption.SPARSE) {
continue;
} else {
throw new IllegalArgumentException("Unsupported open option: " + option);
}
}
return modes;
}
}
}
I am able to fetch the file from the s3 store but nor sure how to read and pass all the contents while someone drag and drop from remote file location to their own system using winSCP. I know i am missing some code at the mentioned place but not sure how to achieve it.

[Ljava.lang.Object; cannot be cast to com.lglsys.entity.EntityName

I was trying to get specific data from database but every time I'm getting the following error!
java.lang.ClassCastException: [Ljava.lang.Object; cannot be cast to com.lglsys.entity.TDasProductDownload
So this is my QueryService class
#Dependent
public class QueryService {
List<TDasProductDownload> downloadLink = new ArrayList();
final private Logger logger =
LogManager.getLogger(QueryService.class.getName());
#PersistenceContext(unitName="DownloadServices")
EntityManager em;
public QueryService() { super(); }
public List<TDasProductDownload> findAllDownloadLinks() {
try {
downloadLink=
em.createQuery(queryForDownloadLinks,TDasProductDownload.class)
.getResultList();
return downloadLink;
} catch (Exception e) {
logger.info(e.toString());
return null;
}
}
}
program gives error in this class /
EndPoint class
public class PreControlWSEndPoint {
private Session session;
final private Logger logger = LogManager.getLogger(PreControlWSEndPoint.class.getName());
List<TDasProductDownload> downloadLink = new ArrayList();
#PersistenceContext(unitName="DownloadServices")
EntityManager em;
#Inject
QueryService service;
#OnOpen
public void Open(Session session) throws IOException, InterruptedException {
this.session = session;
this.sendMessage("Connection Oppened");
logger.info("EndPoint Opened");
try {
downloadLink = service.findAllDownloadLinks();
logger.info(downloadLink.size());
TDasProductDownload str = downloadLink.get(0);
logger.info(str.getDownloadStatus()); //**Eror line!!**
} catch (Exception e) {
logger.info(e.toString() + " .D");
}
}
#OnMessage
public void onMessage(String message) {}
#OnClose
public void Close() {}
}
I can't see what's happening in my code.
I fixed it!
public List<String> findAllDownloadLinks() {
try {
downloadLink=
em.createQuery(queryForDownloadLinks,String.class)
.getResultList();
return downloadLink;
} catch (Exception e) {
logger.info(e.toString());
return null;
}
}
then i can print like so
for(int temp=0;temp<=downloadLink.size();temp++){
logger.info(downloadLink.get(temp));
}

Implementing Checkpointed interface: snapshotState() in flink is not being called

I have created a test below. I am testing snapshots. I supposed that snapshotState and restoreState should be called but it seems that it is not happening. Why?
The main code:
public class CheckpointedTest {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env =
StreamExecutionEnvironment.getExecutionEnvironment();
env.enableCheckpointing(5000);
env.setParallelism(2);
List<Integer> list = new ArrayList<Integer>();
for (int i = 0; i < 10; i++) {
list.add(i);
}
DataStream<Integer> test = env.fromCollection(list);
test.map(new CheckpointedFunction());
env.execute();
}
}
The function implementation is the following:
public class CheckpointedFunction implements MapFunction<Integer, Integer>, Checkpointed<Integer> {
private Integer count = 0;
#Override
public Integer map(Integer value) throws Exception {
System.out.println("count: " + count);
Thread.sleep((long) (Math.random()*4000));
return count++;
}
#Override
public Integer snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {
System.out.println("Snapshot count: " + count);
return count;
}
#Override
public void restoreState(Integer state) throws Exception {
this.count = state;
System.out.println("Restored count: " + count);
}
}

Increasing heap by excessive use oft Java ScriptEngine (Jyhton)

We have a JavaEE application that uses jython to execute some python scripts. By and by the used heapspace gets bigger and bigger until there is no more heapspace left. In a heapdump i can se that there are a lot of Py*-classes.
So i wrote a small test-program:
TestApp
public class TestApp {
private final ScriptEngineManager scriptEngineManager = new ScriptEngineManager();
private HashMap<String, ScriptEngine> scriptEngines = new HashMap<String, ScriptEngine>();
private final String scriptContainerPath = "";
public static void main(String[] args) throws InterruptedException {
int counter = 1;
while(true) {
System.out.println("iteration: " + counter);
TestApp testApp = new TestApp();
testApp.execute();
counter++;
Thread.sleep(100);
}
}
void execute() {
File scriptContainer = new File(scriptContainerPath);
File[] scripts = scriptContainer.listFiles();
if (scripts != null && scripts.length > 0) {
Arrays.sort(scripts, new Comparator<File>() {
#Override
public int compare(File file1, File file2) {
return file1.getName().compareTo(file2.getName());
}
});
for (File script : scripts) {
String engineName = ScriptExecutor.getEngineNameByExtension(script.getName());
if(!scriptEngines.containsKey(engineName)) {
scriptEngines.put(engineName, scriptEngineManager.getEngineByName(engineName));
}
ScriptEngine scriptEngine = scriptEngines.get(engineName);
try {
ScriptExecutor scriptExecutor = new ScriptExecutor(scriptEngine, script, null);
Boolean disqualify = scriptExecutor.getBooleanScriptValue("disqualify");
String reason = scriptExecutor.getStringScriptValue("reason");
System.out.println("disqualify: " + disqualify);
System.out.println("reason: " + reason);
} catch (Exception e) {
e.printStackTrace();
}
}
// cleanup
for(Map.Entry<String, ScriptEngine> entry : scriptEngines.entrySet()) {
ScriptEngine engine = entry.getValue();
engine.getContext().setErrorWriter(null);
engine.getContext().setReader(null);
engine.getContext().setWriter(null);
}
}
}
}
ScriptExecutor
public class ScriptExecutor {
private final static String pythonExtension = "py";
private final static String pythonEngine = "python";
private final ScriptEngine scriptEngine;
public ScriptExecutor(ScriptEngine se, File file, Map<String, Object> keyValues) throws FileNotFoundException, ScriptException {
scriptEngine = se;
if (keyValues != null) {
for (Map.Entry<String, Object> entry : keyValues.entrySet()) {
scriptEngine.put(entry.getKey(), entry.getValue());
}
}
// execute script
Reader reader = null;
try {
reader = new FileReader(file);
scriptEngine.eval(reader);
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e) {
// nothing to do
}
}
}
}
public Boolean getBooleanScriptValue(String key) {
// convert Object to Boolean
}
public String getStringScriptValue(String key) {
// convert Object to String
}
public static String getEngineNameByExtension(String fileName) {
String extension = fileName.substring(fileName.lastIndexOf(".") + 1);
if (pythonExtension.equalsIgnoreCase(extension)) {
System.out.println("Found engine " + pythonEngine + " for extension " + extension + ".");
return pythonEngine;
}
throw new RuntimeException("No suitable engine found for extension " + extension);
}
}
In the specified directory are 14 python scripts that all look like this:
disqualify = True
reason = "reason"
I start this program with the following VM-arguments:
-Xrs -Xms16M -Xmx16M -XX:MaxPermSize=32M -XX:NewRatio=3 -Dsun.rmi.dgc.client.gcInterval=300000 -Dsun.rmi.dgc.server.gcInterval=300000 -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSParallelRemarkEnabled -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -server
These are the arguments our AppServer is running with. Only Xms, Xmx and MaxPermSize are smaller in my testcase.
When I run this application I can see that the CMS Old Gen pool increases to its max size. After that the Par Eden Space pool increases. In addition at any time the ParNewGC does not run anymore. The cleanup part improved the situation but didn't resolve the problem. Has anybody an idea why my heap isn't completly cleaned?
I think I have found a solution for my problem: I removed the JSR223 stuff und now use the PythonInterpreter directly.

Resources