How to set receiveTimeout and connection timeout for cxfEndpoint - cxf

I am trying to set receiveTimeout and connection timeout for cxfEndpoint in below code .. I got so many spring dsl related answer but i am using camel dsl specifically.
I am trying to set receiveTimeout and connection timeout for cxfEndpoint in below code .. I got so many spring dsl related answer but i am using camel dsl specifically.
I am trying to set receiveTimeout and connection timeout for cxfEndpoint in below code .. I got so many spring dsl related answer but i am using camel dsl specifically.
I am trying to set receiveTimeout and connection timeout for cxfEndpoint in below code .. I got so many spring dsl related answer but i am using camel dsl specifically.
void configure() throws Exception {
super.configure()
CamelContext context=getContext()
String version=context.resolvePropertyPlaceholders('{{'+ CommonConstants.VERSION_PROPERTY+ '}}')
String region=context.resolvePropertyPlaceholders('{{'+ CommonConstants.REGION_PROPERTY + '}}')
String getContextRoot=context.resolvePropertyPlaceholders('{{' + CommonConstants.CONTEXT_ROOT_PROPERTY + '}}')
boolean validateResponse=getContextRoot
//main route exposing a GET
rest("/$version/$region/")
.get("/$getContextRoot")
.produces('application/json')\
.to('direct:validate')
from('direct:validate')
.routeId('validate')
.bean(ValidatorSubRouteHelper.class,'validate')
.to('direct:get-deviceIdentification')
from('direct:get-deviceIdentification')
.routeId('get-deviceIdentification')
//pre-processing closure
.process {
it.out.body = [ it.properties[MessageReferenceConstants.USER_AGENT_HEADER], new CallContext() ]
it.in.headers[CxfConstants.OPERATION_NAME] = context.resolvePropertyPlaceholders('{{'+MessageReferenceConstants.PROPERTY_OPERATION_NAME+'}}')
it.in.headers[Exchange.SOAP_ACTION] = context.resolvePropertyPlaceholders('{{'+MessageReferenceConstants.PROPERTY_SOAP_ACTION+'}}')
Map<String, Object> reqCtx = new HashMap<String, Object>();
HTTPClientPolicy clientHttpPolicy = new HTTPClientPolicy();
clientHttpPolicy.setReceiveTimeout(10000);
reqCtx.put(HTTPClientPolicy.class.getName(), clientHttpPolicy)
it.in.headers[Client.REQUEST_CONTEXT]=reqCtx
}
.to(getEndpointURL())
//In case of SOAPFault from device, handling the exception in processSOAPResponse
.onException(SoapFault.class)
.bean(ProcessResponseExceptionHelper.class,"processSOAPResponse")
.end()
//post-processing closure
.process {
log.info("processing the response retrieved from device service")
MessageContentsList li = it.in.getBody(MessageContentsList.class)
DeviceFamily deviceFamily = (DeviceFamily) li.get(0)
log.debug('device type is '+deviceFamily.deviceType.value)
it.properties[MessageReferenceConstants.PROPERTY_RESPONSE_BODY] = deviceFamily.deviceType.value
}.to('direct:transform')
from('direct:transform')
.routeId('transform')
//transform closure
.process {
log.info("Entering the FilterTransformSubRoute(transform)")
Device device=new Device()
log.debug('device type '+it.properties[MessageReferenceConstants.PROPERTY_RESPONSE_BODY])
device.familyName = it.properties[MessageReferenceConstants.PROPERTY_RESPONSE_BODY]
it.out.body=device
}
.choice()
.when(simple('{{validateResponse}}'))
.to('direct:validateResponse')
if(validateResponse) {
from('direct:validateResponse')
.bean(DataValidator.getInstance('device.json'))
}
}
/**
* Constructs the endpoint url.
* Formatting end point URL for device identification service call
* #return the endpoint url
*/
private String getEndpointURL() {
CamelContext context=getContext()
def serviceURL=context.resolvePropertyPlaceholders('{{'+MessageReferenceConstants.SERVICE_URL+'}}')
def wsdlURL=context.resolvePropertyPlaceholders('{{'+MessageReferenceConstants.WSDL_URL+'}}')
boolean isGZipEnable=CommonConstants.TRUE.equalsIgnoreCase(context.resolvePropertyPlaceholders('{{'+MessageReferenceConstants.GZIP_ENABLED+'}}'))
def serviceClass = context.resolvePropertyPlaceholders('{{'+MessageReferenceConstants.PROPERTY_SERVICE_CLASS+'}}')
def serviceName = context.resolvePropertyPlaceholders('{{'+MessageReferenceConstants.PROPERTY_SERVICE_NAME+'}}')
def url="cxf:$serviceURL?"+
"wsdlURL=$wsdlURL"+
"&serviceClass=$serviceClass"+
"&serviceName=$serviceName"
if(isGZipEnable) {
url+= "&cxfEndpointConfigurer=#deviceIdentificationServiceCxfConfigurer"
}
log.debug("endpoint url is " + url)
url
}

You already find a the cxfEndpointConfigurer option for it.
Now you just need implement the configurer interface like this:
public static class MyCxfEndpointConfigurer implements CxfEndpointConfigurer {
#Override
public void configure(AbstractWSDLBasedEndpointFactory factoryBean) {
// Do nothing here
}
#Override
public void configureClient(Client client) {
// reset the timeout option to override the spring configuration one
HTTPConduit conduit = (HTTPConduit) client.getConduit();
HTTPClientPolicy policy = new HTTPClientPolicy();
// You can setup the timeout option here
policy.setReceiveTimeout(60000);
policy.setConnectionTimeout(30000);
conduit.setClient(policy);
}
#Override
public void configureServer(Server server) {
// Do nothing here
}
}

Related

Flink JDBC Sink part 2

I have posted a question few days back- Flink Jdbc sink
Now, I am trying to use the sink provided by flink.
I have written the code and it worked as well. But nothing got saved in DB and no exceptions were there. Using previous sink my code was not finishing(that should happen ideally as its a streaming app) but after the following code I am getting no error and the nothing is getting saved to DB.
public class CompetitorPipeline implements Pipeline {
private final StreamExecutionEnvironment streamEnv;
private final ParameterTool parameter;
private static final Logger LOG = LoggerFactory.getLogger(CompetitorPipeline.class);
public CompetitorPipeline(StreamExecutionEnvironment streamEnv, ParameterTool parameter) {
this.streamEnv = streamEnv;
this.parameter = parameter;
}
#Override
public KeyedStream<CompetitorConfig, String> start(ParameterTool parameter) throws Exception {
CompetitorConfigChanges competitorConfigChanges = new CompetitorConfigChanges();
KeyedStream<CompetitorConfig, String> competitorChangesStream = competitorConfigChanges.run(streamEnv, parameter);
//Add to JBDC Sink
competitorChangesStream.addSink(JdbcSink.sink(
"insert into competitor_config_universe(marketplace_id,merchant_id, competitor_name, comp_gl_product_group_desc," +
"category_code, competitor_type, namespace, qualifier, matching_type," +
"zip_region, zip_code, competitor_state, version_time, compConfigTombstoned, last_updated) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
(ps, t) -> {
ps.setInt(1, t.getMarketplaceId());
ps.setLong(2, t.getMerchantId());
ps.setString(3, t.getCompetitorName());
ps.setString(4, t.getCompGlProductGroupDesc());
ps.setString(5, t.getCategoryCode());
ps.setString(6, t.getCompetitorType());
ps.setString(7, t.getNamespace());
ps.setString(8, t.getQualifier());
ps.setString(9, t.getMatchingType());
ps.setString(10, t.getZipRegion());
ps.setString(11, t.getZipCode());
ps.setString(12, t.getCompetitorState());
ps.setTimestamp(13, Timestamp.valueOf(t.getVersionTime()));
ps.setBoolean(14, t.isCompConfigTombstoned());
ps.setTimestamp(15, new Timestamp(System.currentTimeMillis()));
System.out.println("sql"+ps);
},
new JdbcConnectionOptions.JdbcConnectionOptionsBuilder()
.withUrl("jdbc:mysql://127.0.0.1:3306/database")
.withDriverName("com.mysql.cj.jdbc.Driver")
.withUsername("xyz")
.withPassword("xyz#")
.build()));
return competitorChangesStream;
}
}
You need enable autocommit mode for jdbc Sink.
new JdbcConnectionOptions.JdbcConnectionOptionsBuilder()
.withUrl("jdbc:mysql://127.0.0.1:3306/database;autocommit=true")
It looks like SimpleBatchStatementExecutor only works in auto-commit mode. And if you need to commit and rollback batches, then you have to write your own ** JdbcBatchStatementExecutor **
Have you tried to include the JdbcExecutionOptions ?
dataStream.addSink(JdbcSink.sink(
sql_statement,
(statement, value) -> {
/* Prepared Statement */
},
JdbcExecutionOptions.builder()
.withBatchSize(5000)
.withBatchIntervalMs(200)
.withMaxRetries(2)
.build(),
new JdbcConnectionOptions.JdbcConnectionOptionsBuilder()
.withUrl("jdbc:mysql://127.0.0.1:3306/database")
.withDriverName("com.mysql.cj.jdbc.Driver")
.withUsername("xyz")
.withPassword("xyz#")
.build()));

is JSONDeserializationSchema() deprecated in Flink?

I am new to Flink and doing something very similar to the below link.
Cannot see message while sinking kafka stream and cannot see print message in flink 1.2
I am also trying to add JSONDeserializationSchema() as a deserializer for my Kafka input JSON message which is without a key.
But I found JSONDeserializationSchema() is not present.
Please let me know if I am doing anything wrong.
JSONDeserializationSchema was removed in Flink 1.8, after having been deprecated earlier.
The recommended approach is to write a deserializer that implements DeserializationSchema<T>. Here's an example, which I've copied from the Flink Operations Playground:
import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
/**
* A Kafka {#link DeserializationSchema} to deserialize {#link ClickEvent}s from JSON.
*
*/
public class ClickEventDeserializationSchema implements DeserializationSchema<ClickEvent> {
private static final long serialVersionUID = 1L;
private static final ObjectMapper objectMapper = new ObjectMapper();
#Override
public ClickEvent deserialize(byte[] message) throws IOException {
return objectMapper.readValue(message, ClickEvent.class);
}
#Override
public boolean isEndOfStream(ClickEvent nextElement) {
return false;
}
#Override
public TypeInformation<ClickEvent> getProducedType() {
return TypeInformation.of(ClickEvent.class);
}
}
For a Kafka producer you'll want to implement KafkaSerializationSchema<T>, and you'll find examples of that in that same project.
To solve the problem of reading non-key JSON messages from Kafka I used case class and JSON parser.
The following code makes a case class and parses the JSON field using play API.
import play.api.libs.json.JsValue
object CustomerModel {
def readElement(jsonElement: JsValue): Customer = {
val id = (jsonElement \ "id").get.toString().toInt
val name = (jsonElement \ "name").get.toString()
Customer(id,name)
}
case class Customer(id: Int, name: String)
}
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
val properties = new Properties()
properties.setProperty("bootstrap.servers", "xxx.xxx.0.114:9092")
properties.setProperty("group.id", "test-grp")
val consumer = new FlinkKafkaConsumer[String]("customer", new SimpleStringSchema(), properties)
val stream1 = env.addSource(consumer).rebalance
val stream2:DataStream[Customer]= stream1.map( str =>{Try(CustomerModel.readElement(Json.parse(str))).getOrElse(Customer(0,Try(CustomerModel.readElement(Json.parse(str))).toString))
})
stream2.print("stream2")
env.execute("This is Kafka+Flink")
}
The Try method lets you overcome the exception thrown while parsing the data
and returns the exception in one of the fields (if we want) or else it can just return the case class object with any given or default fields.
The sample output of the Code is:
stream2:1> Customer(1,"Thanh")
stream2:1> Customer(5,"Huy")
stream2:3> Customer(0,Failure(com.fasterxml.jackson.databind.JsonMappingException: No content to map due to end-of-input
at [Source: ; line: 1, column: 0]))
I am not sure if it is the best approach but it is working for me as of now.

camel route test - the registry for: mock:result

Hi I have complex camel route and in between the route I am sending mesage to MQ using Bean.
.bean("{{tp.mqservice}}")
application.yaml
mqservice: bean:mqService
application-test.yaml
mqservice: mock:result
Below is my PortfolioRouteTest
#ActiveProfiles("test")
#RunWith(CamelSpringBootRunner.class)
#SpringBootTest(classes = MainApplication.class, webEnvironment = WebEnvironment.RANDOM_PORT)
#MockEndpoints
public class PortfolioTncRouteTest {
#EndpointInject(value = "{{trade-publisher.portfolio-tnc.source-endpoint}}")
private ProducerTemplate producerTemplate;
#EndpointInject(value = "{{trade-publisher.mqservice}}")
private MockEndpoint mock;
}
Junit
#Test
public void portfolioTncRouteTest() throws InterruptedException {
data = ...
Mockito.when(service.search(Mockito.any(....class))).thenReturn(...);
producerTemplate.sendBody(data);
mock.expectedMessageCount(1);
mock.assertIsSatisfied(30000);
}
however when I run the test I am getting below error. Am I missing something?
Stacktrace
Caused by: org.apache.camel.NoSuchBeanException: No bean could be found in the registry for: mock:result
at org.apache.camel.component.bean.RegistryBean.getBean(RegistryBean.java:92)
at org.apache.camel.component.bean.RegistryBean.createCacheHolder(RegistryBean.java:67)
at org.apache.camel.reifier.BeanReifier.createProcessor(BeanReifier.java:57)
at org.apache.camel.reifier.ProcessorReifier.createProcessor(ProcessorReifier.java:485)
at org.apache.camel.reifier.ProcessorReifier.createOutputsProcessorImpl(ProcessorReifier.java:448)
at org.apache.camel.reifier.ProcessorReifier.createOutputsProcessor(ProcessorReifier.java:415)
at org.apache.camel.reifier.ProcessorReifier.createOutputsProcessor(ProcessorReifier.java:212)
at org.apache.camel.reifier.ExpressionReifier.createFilterProcessor(ExpressionReifier.java:39)
at org.apache.camel.reifier.WhenReifier.createProcessor(WhenReifier.java:32)
at org.apache.camel.reifier.WhenReifier.createProcessor(WhenReifier.java:24)
at org.apache.camel.reifier.ProcessorReifier.createProcessor(ProcessorReifier.java:485)
at org.apache.camel.reifier.ChoiceReifier.createProcessor(ChoiceReifier.java:54)
at org.apache.camel.reifier.ProcessorReifier.createProcessor(ProcessorReifier.java:485)
at org.apache.camel.reifier.ProcessorReifier.createOutputsProcessorImpl(ProcessorReifier.java:448)
at org.apache.camel.reifier.ProcessorReifier.createOutputsProcessor(ProcessorReifier.java:415)
at org.apache.camel.reifier.TryReifier.createProcessor(TryReifier.java:38)
at org.apache.camel.reifier.ProcessorReifier.createProcessor(ProcessorReifier.java:485)
at org.apache.camel.reifier.ProcessorReifier.createOutputsProcessorImpl(ProcessorReifier.java:448)
at org.apache.camel.reifier.ProcessorReifier.createOutputsProcessor(ProcessorReifier.java:415)
at org.apache.camel.reifier.ProcessorReifier.createOutputsProcessor(ProcessorReifier.java:212)
at org.apache.camel.reifier.ProcessorReifier.createChildProcessor(ProcessorReifier.java:231)
at org.apache.camel.reifier.SplitReifier.createProcessor(SplitReifier.java:42)
at org.apache.camel.reifier.ProcessorReifier.makeProcessorImpl(ProcessorReifier.java:536)
at org.apache.camel.reifier.ProcessorReifier.makeProcessor(ProcessorReifier.java:497)
at org.apache.camel.reifier.ProcessorReifier.addRoutes(ProcessorReifier.java:241)
at org.apache.camel.reifier.RouteReifier.addRoutes(RouteReifier.java:358)
... 56 more
Use .to instead of .bean so its sending to a Camel endpoint, then you can send to the mock endpoint. When using .bean then its for calling a POJO Java bean only.

Setting connection timeout in RestEASY

JBoss tells us
http://docs.jboss.org/seam/3/rest/latest/reference/en-US/html/rest.client.html
that to set the timeout for a RestEASY ClientRequest we must create a custom ClientExecutor, then call deprecated static methods on ConnManagerParams. This seems rather hokey. Is there a better way? This is RestEASY 2.3.6.
Here is a clean working solution :-)
#Singleton
public class RestEasyConfig {
#Inject
#MyConfig
private Integer httpClientMaxConnectionsPerRoute;
#Inject
#MyConfig
private Integer httpClientTimeoutMillis;
#Inject
#MyConfig
private Integer httpClientMaxTotalConnections;
#Produces
private ClientExecutor clientExecutor;
#PostConstruct
public void createExecutor() {
final BasicHttpParams params = new BasicHttpParams();
HttpConnectionParams.setConnectionTimeout(params, this.httpClientTimeoutMillis);
final SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
final ThreadSafeClientConnManager connManager = new ThreadSafeClientConnManager(schemeRegistry);
connManager.setDefaultMaxPerRoute(this.httpClientMaxConnectionsPerRoute);
connManager.setMaxTotal(this.httpClientMaxTotalConnections);
final HttpClient httpClient = new DefaultHttpClient(connManager, params);
this.clientExecutor = new ApacheHttpClient4Executor(httpClient);
}
}
With RestEASY 3.12.1.Final, as explained by redhat website, I did it this way:
private Client clientBuilder() {
return new ResteasyClientBuilder()
.connectTimeout(2, TimeUnit.SECONDS)
.readTimeout(10, TimeUnit.SECONDS)
.build()
.register(ClientRestLoggingFilter.class)
.register(ObjectMapperContextResolver.class);
}

Setting a header based on an XQuery filter

I have a route that's set to run in batched mode, polling several thousand XML files. Each is timestamped inside the XML structure and this dateTime element is used to determine whether the XML should be included in the batch's further processing (an XQuery transform). As this is a batch route it self-terminates after execution.
Because the route needs to close itself I have to ensure that it also closes if every message is filtered out, which is why I don't use a filter but a .choice() statement instead and set a custom header on the exchange which is later used in a bean that groups matches and prepares a single source document for the XQuery.
However, my current approach requires a second route that both branches of the .choice() forward to. This is necessary because I can't seem to force both paths to simply continue. So my question is: how can get rid of this second route? One approach is setting the filter header in a bean instead but I'm worried about the overhead involved. I assume the XQuery filter inside Camel would greatly outperform a POJO that builds an XML document from a string and runs an XQuery against it.
from(sourcePath + "?noop=true" + "&include=.*.xml")
.choice()
.when()
.xquery("[XQuery Filter]")
.setHeader("Filtered", constant(false))
.to("direct:continue")
.otherwise()
.setHeader("Filtered", constant(true))
.to("direct:continue")
.end();
from("direct:continue")
.routeId(forwarderRouteID)
.aggregate(aggregationExpression)
.completionFromBatchConsumer()
.completionTimeout(DEF_COMPLETION_TIMEOUT)
.groupExchanges()
.bean(new FastQueryMerger(), "group")
.to("xquery:" + xqueryPath)
.bean(new FileModifier(interval), "setFileName")
.to(targetPath)
.process(new Processor() {
#Override
public void process(Exchange exchange) throws Exception {
new RouteTerminator(routeID, exchange.getContext()).start();
new RouteTerminator(forwarderRouteID, exchange.getContext()).start();
}
})
.end();
Wouldn't .end() help here?
I mean the following:
from(sourcePath + "?noop=true" + "&include=.*.xml")
.choice()
.when()
.xquery("[XQuery Filter]")
.setHeader("Filtered", constant(false)).end()
.otherwise()
.setHeader("Filtered", constant(true)).end()
.aggregate(aggregationExpression)
.completionFromBatchConsumer()
.completionTimeout(DEF_COMPLETION_TIMEOUT)
.groupExchanges()
.bean(new FastQueryMerger(), "group")
.to("xquery:" + xqueryPath)
.bean(new FileModifier(interval), "setFileName")
.to(targetPath)
.process(new Processor() {
#Override
public void process(Exchange exchange) throws Exception {
new RouteTerminator(routeID, exchange.getContext()).start();
new RouteTerminator(forwarderRouteID, exchange.getContext()).start();
}
});
just quickly tested the following one and it worked:
#Produce(uri = "direct:test")
protected ProducerTemplate testProducer;
#EndpointInject(uri = "mock:test-first")
protected MockEndpoint testFirst;
#EndpointInject(uri = "mock:test-therest")
protected MockEndpoint testTheRest;
#EndpointInject(uri = "mock:test-check")
protected MockEndpoint testCheck;
#Test
public void test() {
final String first = "first";
final String second = "second";
testFirst.setExpectedMessageCount(1);
testTheRest.setExpectedMessageCount(1);
testCheck.setExpectedMessageCount(2);
testProducer.sendBody(first);
testProducer.sendBody(second);
try {
testFirst.assertIsSatisfied();
testTheRest.assertIsSatisfied();
testCheck.assertIsSatisfied();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
#Override
protected RouteBuilder createRouteBuilder() {
return new RouteBuilder() {
public void configure() {
from("direct:test")
.choice()
.when(body().isEqualTo("first")).to("mock:test-first")
.otherwise().to("mock:test-therest").end()
.to("mock:test-check");
}
};
}

Resources