I've got a Flink KeyedCoProcessFunction that registers Processing Time Timers in a larger Flink stream job, and I'm trying to create unit tests for the entire job using the Flink MiniCluster. But I can't get the onTimer() call back in the KeyedCoProcessFunction to trigger.
Has anyone gotten this to work? Did it require any special configuration?
Switching to Event Time works fine, so I'm wondering if this just doesn't work with the Flink MiniCluster or is there something wrong with my implementation.
I wrote a simple test in Scala to see if I could get this to work.
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.KeyedProcessFunction
import org.apache.flink.streaming.api.functions.source.{ParallelSourceFunction, SourceFunction}
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.test.streaming.runtime.util.TestListResultSink
import org.apache.flink.test.util.MiniClusterWithClientResource
import org.apache.flink.util.Collector
import org.scalatest.BeforeAndAfter
import org.scalatest.flatspec.AnyFlatSpec
import org.slf4j.LoggerFactory
class TimerTest extends AnyFlatSpec with BeforeAndAfter {
private val SlotsPerTaskMgr = 1
val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder()
.setNumberSlotsPerTaskManager(SlotsPerTaskMgr)
.setNumberTaskManagers(1)
.build)
before {
flinkCluster.before()
}
after {
flinkCluster.after()
}
"MiniCluster" should "trigger onTimer" in {
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime)
implicit val longTypeInfo: TypeInformation[Long] = TypeInformation.of(classOf[Long])
val sink = new TestListResultSink[Long]
env.addSource(new MyLongSource(100))
.keyBy(v => v)
.process(new MyProccesor())
.addSink(sink)
env.execute()
println("Received " + sink.getResult.size() + " output records.")
}
}
class MyProccesor extends KeyedProcessFunction[Long, Long, Long] {
private val log = LoggerFactory.getLogger(this.getClass)
override def processElement(
value: Long,
ctx: KeyedProcessFunction[Long, Long, Long]#Context,
out: Collector[Long]): Unit = {
log.info("Received {} at {}", value, ctx.timerService().currentProcessingTime())
if (value % 10 == 0) {
log.info("Scheduling processing timer for {}", ctx.timerService().currentProcessingTime() + 10)
ctx.timerService().registerProcessingTimeTimer(ctx.timerService().currentProcessingTime() + 10)
}
}
override def onTimer(
timestamp: Long,
ctx: KeyedProcessFunction[Long, Long, Long]#OnTimerContext,
out: Collector[Long]): Unit = {
log.info("Received onTimer at {}", timestamp)
out.collect(timestamp)
}
}
class MyLongSource(n:Int) extends ParallelSourceFunction[Long] {
#volatile private var stop = false
override def run(ctx: SourceFunction.SourceContext[Long]): Unit = {
for(i <- 1 to n) {
if(stop) return;
println("Sending " + i)
ctx.collect(i)
}
Thread.sleep(1000)
}
override def cancel(): Unit = {
stop = true
}
}
I was finally able to get some consistent results by adding a Thread.sleep(1000) at the end of the source run() method. Seems like once the source exits, messages get processed and then everything is shut down even if there are pending timers.
When a Flink job shuts down, any pending processing time timers are simply ignored. They never fire.
For what it's worth, there's some ongoing discussion on the Flink dev mailing list about offering an option to trigger all pending processing time timers. See http://apache-flink-user-mailing-list-archive.2336050.n4.nabble.com/DISCUSS-FLIP-134-DataStream-Semantics-for-Bounded-Input-td37365.html#a37558.
Related
Im fairly new into programming, i was trying to launch my test but i got : Exception in thread "main" java.lang.IllegalArgumentException: requirement failed: No scenario set up
even if i got the scenario set up (which is confusing)
i'm pretty sure is something obvious that newbie like me can't figure out unfortunately.
i tried invalidating caches, and reloading the project.
im using intelij + maven
package simulations
import io.gatling.core.Predef._
import io.gatling.core.structure.{ChainBuilder, ScenarioBuilder}
import io.gatling.http.Predef._
import io.gatling.http.protocol.HttpProtocolBuilder
class GatlingDemoStore extends Simulation {
val domain = "demostore.gatling.io"
val httpProtocol: HttpProtocolBuilder = http
.baseUrl("http://" + domain)
object login {
def userlogin: ChainBuilder = {
exec(http("Login User")
.post("/login")
.formParam("_csrf", "${csrfValue}")
.formParam("username", "user1")
.formParam("password", "pass"))
}
}
object CmsPages {
def homepage: ChainBuilder = {
exec(http("Load Home Page")
.get("/")
.check(status.is(200))
.check(regex("<title>Gatling Demo-Store</title>").exists)
.check(css("#_csrf", "content").saveAs("csrfValue")))
}
def aboutus: ChainBuilder = {
exec(http("Load Home Page")
.get("/about-us")
.check(status.is(200))
.check(substring("About Us")))
}
def categories: ChainBuilder = {
exec(http("Load Categories Page")
.get("/category/all")
.check(status.is(200)))
.pause(10)
}
def productpage: ChainBuilder = {
exec(http("Load Product Page")
.get("/product/black-and-red-glasses")
.check(status.is(200)))
.pause(15)
}
def addtocart: ChainBuilder = {
exec(http("Add Product to Cart")
.get("/cart/add/19"))
}
def viewcart: ChainBuilder = {
exec(http("View Cart")
.get("/cart/view"))
}
def checkout: ChainBuilder = {
exec(http("Checkout")
.get("/cart/checkout"))
}
val User: ScenarioBuilder = scenario("DemoStore Simulation")
.exec(CmsPages.homepage)
.pause(5)
.exec(CmsPages.aboutus)
.pause(5)
.exec(CmsPages.categories)
.pause(20)
.exec(CmsPages.productpage)
.pause(5)
.exec(CmsPages.addtocart)
.pause(2)
.exec(login.userlogin)
setUp(
User.inject(atOnceUsers(1))
).protocols(httpProtocol)
}
}
I've reformatted your code so your error is more obvious: you're calling setUp not in the body of the GatlingDemoStore class but in the body of the CmsPages object, which is never loaded (in Scala, objects are lazily loaded and here you never call it).
Move setUp in the body of the GatlingDemoStore class.
Important note: as you're new to programming, you should probably go with Java (supported since Gatling 3.7) instead of Scala.
thank you very much for your help, greatly appreciated.
seems like a rookie mistake..
thats a great idea actually, ima do some research on how i can do that.
I was wondering how to make my app save data after restart? (The user can delete task and add new task to list, as well as check the box that the task is done. I want the app to save this data so when the user exists the app it will display all the tasks that he left the app with)
I was reading on google for few hours now, I got to
[1]: https://flutter.dev/docs/cookbook/persistence/reading-writing-files
This link as someone recommended on a similar post. But after reading it through I am a bit confused about where to start with my app.
Including some of my code and if you could help me I would really appreciate it as after hours of reading and watching tutorials I am still quite unsure where to start or which way of doing this is best.
My main.dart is this
void main() {
runApp(MyApp());
}
class MyApp extends StatelessWidget {
#override
Widget build(BuildContext context) {
return ChangeNotifierProvider(
create: (context) =>
TaskData(), //changing builder: to create: fixed the errors i been having
child: MaterialApp(
home: TasksScreen(),
),
);
}
}
class TaskData extends ChangeNotifier {
List<Task> _tasks = [
Task(name: "Sample task 1"),
Task(name: "Sample task 2"),
Task(name: "Sample task 3"),
];
UnmodifiableListView<Task> get tasks {
return UnmodifiableListView(_tasks);
}
int get taskCount {
return _tasks.length;
}
void addTask(String newTaskTitle) {
final task = Task(name: newTaskTitle);
_tasks.add(task);
notifyListeners();
}
void updateTask(Task task) {
task.toggleDone();
notifyListeners();
}
void deleteTask(Task task) {
_tasks.remove(task);
notifyListeners();
}
Thank you so much!
The basic method is using the device local storage.
1 - Add the shared_preferences in your pubspec.yaml
2 - Create a class to write and read data :
import 'dart:convert';
import 'package:shared_preferences/shared_preferences.dart';
class StoreData {
StoreData._privateConstructor();
static final StoreData instance = StoreData._privateConstructor();
Future<void> saveString(String key, String value) async {
try{
SharedPreferences pref = await SharedPreferences.getInstance();
final encodedValue = base64.encode(utf8.encode(value));
pref.setString(key, encodedValue);
} catch (e){
print('saveString ${e.toString()}');
}
}
Future<String> getString(String key) async {
SharedPreferences pref = await SharedPreferences.getInstance();
final value = pref.getString(key) == null ? '' : pref.getString(key);
if (value.length > 0) {
final decodedValue = utf8.decode(base64.decode(value));
return decodedValue.toString();
}
return '';
}
Future<bool> remove(String key) async {
SharedPreferences pref = await SharedPreferences.getInstance();
return pref.remove(key);
}
}
3 Use :
Save Data:
StoreData.instance.saveString('name', 'sergio');
Retrieve Data:
final String storedName = await StoreData.instance.getString('name');
print('The name is $storedName');
We have many other methods, like use a SQlite, NoSql or a Database in back-end, but the local storage is the most basic case
What you need is a database or a document based data storage. You can store data in a local sqlite db, using sqflite plugin. Or you can store in a JSON asset file.
You can also use a server or cloud service. Firebase is pretty well integrated with flutter, but AWS and Azure are also great.
You can write the data in a text file in the asset, but that would very complicated.
READY! Here it is : We have a simple but useful code that loads 2 swf files in sequence ...
But How can I Loop it?
How can you change the code to load swf1 after swf2 is finished?
I've tried almost the whole day but no result yet... Please help...Even with your comments any any any idea is greatly appreciated.
Thanks a lot...
Here is the code:
import com.greensock.*;
import com.greensock.loading.*;
import com.greensock.events.LoaderEvent;
import flash.events.Event;
//create SWFLoaders
var swf1:SWFLoader = new SWFLoader("child1.swf",{container:this,y:100,onProgress:progressHandler,onComplete:completeHandler,autoPlay:false});
var swf2:SWFLoader = new SWFLoader("child2.swf",{container:this,y:100,onProgress:progressHandler,onComplete:completeHandler,autoPlay:false});
var currentSWFLoader:SWFLoader = swf1;
//adjust the progress bar;
function progressHandler(e:LoaderEvent):void {
bar.scaleX = e.target.progress;
trace(e.target.progress);
}
//tell the loaded swf to play and start tracking frames
function completeHandler(e:LoaderEvent):void {
//the target of the LoaderEvent is the SWFLoader that fired the event
//the rawContent is the loaded swf
e.target.rawContent.play();
addEventListener(Event.ENTER_FRAME, checkFrame);
}
function checkFrame(e:Event):void {
//check to see if loaded swf is done playing
if (currentSWFLoader.rawContent.currentFrame == currentSWFLoader.rawContent.totalFrames) {
trace("swf done playing");
removeEventListener(Event.ENTER_FRAME, checkFrame);
//if the first swf is done playing load the second swf
if (currentSWFLoader == swf1) {
currentSWFLoader.dispose(true) // dispose and unload content
currentSWFLoader = swf2;
currentSWFLoader.load();
}
}
}
bar.scaleX = 0;
currentSWFLoader.load();
You need OOP approach.
First, create a class that completely handles one external SWF: loading, playing, signaling the end of playback, unloading and cleaning up.
package
{
// Imports.
import flash.events.Event;
import flash.net.URLRequest;
import flash.display.Loader;
import flash.display.Sprite;
import flash.display.MovieClip;
public class ExternalMovie extends Sprite
{
private var L:Loader;
private var M:MovieClip;
// Handles loading.
public function loadAndPlay(url:String):void
{
L = new Loader;
addChild(L);
L.contentLoaderInfo.addEventListener(Event.INIT, onLoaded);
L.load(new URLRequest(url));
}
// Handles Loader.contentLoaderInfo event Event.INIT.
private function onLoaded(e:Event):void
{
L.contentLoaderInfo.removeEventListener(Event.INIT, onLoaded);
// Get a direct reference to the loaded movie root.
M = L.content as MovieClip;
// Set up the watchdog for loaded movie to finish.
addEventListener(Event.ENTER_FRAME, onWatch);
}
// Event.ENTER_FRAME handler to watch the loaded movie playback.
private function onWatch(e:Event):void
{
if (M.currentFrame >= M.totalFrames)
{
M.stop();
// Announce the end of playback.
dispatchEvent(new Event(Event.COMPLETE));
}
}
// Unloads movie and cleans up.
public function dispose():void
{
removeEventListener(Event.ENTER_FRAME, onWatch);
L.unloadAndStop(true);
removeChild(L);
L = null;
M = null;
}
}
}
Second, compose a controlling script, that makes an instance of that class, tells it to play SWF by its URL, then waits for the end of playback and moves on to the next movie.
import ExternalMovie;
var EML:ExternalMovie;
var current:int = -1;
var plan:Array = ["child1.swf", "child2.swf"];
// Start the first one manually.
playNext();
function playNext(e:Event = null):void
{
// Clean up the last one, if any.
if (EML)
{
removeChild(EML);
EML.removeEventListener(Event.COMPLETE, playNext);
EML.dispose();
EML = null;
}
// Switch to next url.
current++;
// Loop from the beginning if last movie just played.
if (current >= plan.length)
{
current = 0;
}
// Get the next movie URL.
var nextURL:String = plan[current];
EML = new ExternalMovie;
// Subscribe to Event.COMPLETE event to know the exact moment the
// current movie finished playing to remove it and load the next one.
EML.addEventListener(Event.COMPLETE, playNext);
EML.loadAndPlay(nextURL);
addChild(EML);
}
Keep in mind that all of above is more of a guideline than a complete working solution (although it just might work).
I am trying to create the Environment in Silhouette but am unable to. I have defined the Identity and Authenticator as follows
trait SessionEnv extends Env {
type I = User
type A = SessionAuthenticator
}
Next, I suppose I have to create the Environment. For that I have written the following code but am stick as I do not understand how to pass the different parameters expected by Environment's apply method
Environment companion object's apply method has signature
def apply[E <: Env](
identityServiceImpl: IdentityService[E#I],
authenticatorServiceImpl: AuthenticatorService[E#A],
requestProvidersImpl: Seq[RequestProvider],
eventBusImpl: EventBus
I know that I have to provide the implemention of IdentityService. I have done so as follows
class UserService #Inject()(userDao:UsersRepository) extends IdentityService[User] {...}
User is defined as follows
case class UserProfile(
loginInfo:LoginInfo,
confirmed: Boolean,
email:Option[String],
firstName: Option[String],
lastName: Option[String],
passwordInfo:Option[PasswordInfo]
//oauth1Info: Option[OAuth1Info],
//avatarUrl: Option[String]) {
)
//representation of a user. A user has an Id and a profile
case class User (id:UUID, profile:UserProfile)
But what do I pass for other values required by apply - authenticatorServiceImpl: AuthenticatorService[E#A],
requestProvidersImpl: Seq[RequestProvider],
eventBusImpl: EventBus
val sessionEnv = com.mohiva.play.silhouette.api.Environment[SessionEnv](new UserService(userRepository),????)
Also, I suppose I don't have to use Guice as I am using compile time injection. Is that correct?
UPDATE
I changed from SessionAuthenticatorService to CookieAuthenticatorService to try some code available online.
It seems my understanding that Silhouette provides some default implementations isn't exactly correct. I thought that I could simply use the SessionAuthenticatorService companion object defined in https://github.com/mohiva/play-silhouette/blob/master/silhouette/app/com/mohiva/play/silhouette/impl/authenticators/SessionAuthenticator.scala but that is not the case. Looking at some code created in ScalaModule, it seems that I'll have to create the required object myself but I'll need to do it in my AppLoader class (for compile time DI) instead of ScalaModule (for runtime DI). However, I still have not solved the problem. I do not know how to create the signer required by CookieAuthenticatorService
val config = configuration.underlying.asInstanceOf[CookieAuthenticatorSettings]("silhouette.authenticator")
val fingerprintGenerator = new DefaultFingerprintGenerator(false)
val idGenerator = new SecureRandomIDGenerator()
val clock:Clock = Clock()
val authenticatorService: AuthenticatorService[CookieAuthenticator] = new CookieAuthenticatorService(config,None,,,fingerprintGenerator, idGenerator,clock) //STILL NEED TO FIND OUT HOW TO CREATE Signer AND CookieHeaderEncoding required by CookieAuthenticator service
val cookieEnv = com.mohiva.play.silhouette.api.Environment[CookieEnv](userIdentityService ,authenticatorService,Seq(),EventBus())
Here is the implementation for cookie authenticator
import com.mohiva.play.silhouette.api.actions._
import com.mohiva.play.silhouette.api.{EventBus, SilhouetteProvider}
import com.mohiva.play.silhouette.api.crypto.CrypterAuthenticatorEncoder
import com.mohiva.play.silhouette.api.util.Clock
import com.mohiva.play.silhouette.crypto.{JcaCrypter, JcaCrypterSettings, JcaSigner, JcaSignerSettings}
import com.mohiva.play.silhouette.impl.authenticators.{CookieAuthenticatorService, CookieAuthenticatorSettings}
import com.mohiva.play.silhouette.impl.util.{DefaultFingerprintGenerator, SecureRandomIDGenerator}
import components._
import play.api.mvc.DefaultCookieHeaderEncoding
import controllers._
import play.api._
import services.db.cassandra.UserService
import play.filters.csrf._
import services.AtomicCounter
import play.api.ApplicationLoader.Context
import play.filters.HttpFiltersComponents
import router.Routes
class AppLoader extends ApplicationLoader {
override def load(context: ApplicationLoader.Context): Application = {
LoggerConfigurator(context.environment.classLoader).foreach {
_.configure(context.environment, context.initialConfiguration, Map.empty)
}
new AppComponents(context).application
}
}
class AppComponents (context: Context) extends BuiltInComponentsFromContext(context)
with CassandraRepositoryComponents
with HttpFiltersComponents
with AssetsComponents
with CSRFComponents
/*with SilhouetteComponents*/{ //TODOM - Would prefer SilhouetteComponent but creating an Environment requires IdentityService. UserService is an IdentifyService but it requires userRepository which is created here. Need to resolve this cross dependence
val userIdentityService = new UserService(userRepository) //responsible for retrieving user information (eg email id) from a database
val config = CookieAuthenticatorSettings()
val fingerprintGenerator = new DefaultFingerprintGenerator(false)
val idGenerator = new SecureRandomIDGenerator()
val clock:Clock = Clock()
val signer= new JcaSigner(new JcaSignerSettings("someSigner"))
val crypter = new JcaCrypter(new JcaCrypterSettings("someCrypter"))
val authenticatorEncoder = new CrypterAuthenticatorEncoder(crypter)
val cookieHeaderEncoding= new DefaultCookieHeaderEncoding()
val authenticatorService = new CookieAuthenticatorService(config, None, signer, cookieHeaderEncoding, authenticatorEncoder, fingerprintGenerator, idGenerator, clock)
val cookieEnv = com.mohiva.play.silhouette.api.Environment[CookieEnv](userIdentityService ,authenticatorService,Seq(),EventBus())
val defaultParser = new mvc.BodyParsers.Default()
val securedAction = new DefaultSecuredAction(new DefaultSecuredRequestHandler(new DefaultSecuredErrorHandler(messagesApi)), defaultParser )
val unsecuredAction = new DefaultUnsecuredAction(new DefaultUnsecuredRequestHandler(new DefaultUnsecuredErrorHandler(messagesApi)),defaultParser)
val userAware = new DefaultUserAwareAction(new DefaultUserAwareRequestHandler(),defaultParser)
val silhouette = new SilhouetteProvider[CookieEnv](cookieEnv,securedAction,unsecuredAction,userAware)
lazy val userRepositoryController = new UserController(userRepository, controllerComponents)
lazy val homeController = new HomeController(controllerComponents, csrfAddToken,csrfCheck,silhouette) //using Silhouette in only one controller for the moment
lazy val countController = new CountController(controllerComponents,new AtomicCounter())
lazy val asyncController = new AsyncController(controllerComponents, actorSystem)
lazy val userWSRoutes = new WSRouters.User.UserRouter(userRepositoryController) //TODOM - whatam i doing here?
lazy val router = new Routes(httpErrorHandler, homeController,userWSRoutes, countController,asyncController, assets)
}
I'm using play-slick_2.11-1.0.1 + HikariCP 2.4.1 to access SqlServer in my Play4-based application.
The database connection in application.conf:
slick.dbs.myDatabase = {
driver="com.typesafe.slick.driver.ms.SQLServerDriver$"
db{
url = "jdbc:sqlserver://sqlserverhost"
driver = com.microsoft.sqlserver.jdbc.SQLServerDriver
user = "admin"
password = "ENCRYPTED_PASSWORD"
}
}
The problem is that the database password configured here must be encrypted based on our company policy.
How can I inject my decryption code to decrypt the password for the connection?
Just found a solution:
def createDecryptedDbConfig (dbConfigProvider: DatabaseConfigProvider) : DatabaseConfig[JdbcProfile] = {
val dbConfig = dbConfigProvider.get[JdbcProfile]
val decryptedConfig = dbConfig.config.
withValue("db.user", ConfigValueFactory.fromAnyRef(decrypt(dbConfig.config.getConfig("db").getString("user")))).
withValue("db.password", ConfigValueFactory.fromAnyRef(decrypt(dbConfig.config.getConfig("db").getString("password"))))
DatabaseConfig.forConfig[JdbcProfile]("", decryptedConfig)
}
i was also trying with latest play 2.7 with slick 3.5
package models
import com.typesafe.config.ConfigValueFactory
import javax.inject.Inject
import play.api.Logger
import play.api.db.slick.DatabaseConfigProvider
import services.EncryptDecryptService
import slick.basic.{BasicProfile, DatabaseConfig}
import slick.jdbc.JdbcProfile
trait HasDatabaseConfigTalachitas[P <: BasicProfile] {
/** The Slick database configuration. */
var dbConfig: DatabaseConfig[P] // field is declared as a val because we want a stable identifier.
/** The Slick profile extracted from `dbConfig`. */
protected final lazy val profile: P = dbConfig.profile // field is lazy to avoid early initializer problems.
#deprecated("Use `profile` instead of `driver`", "2.1")
protected final lazy val driver: P = dbConfig.profile // field is lazy to avoid early initializer problems.
/** The Slick database extracted from `dbConfig`. */
protected final def db: P#Backend#Database = dbConfig.db
}
trait HasDatabaseConfigProviderTalachitas[P <: BasicProfile] extends HasDatabaseConfigTalachitas[P] {
/** The provider of a Slick `DatabaseConfig` instance.*/
protected val dbConfigProvider: DatabaseConfigProvider
override var dbConfig: DatabaseConfig[P] = dbConfigProvider.get[P] // field is lazy to avoid early initializer problems.
}
import play.api.Configuration
class CustomizedSlickConfig #Inject()(config: Configuration,
encryptDecryptService: EncryptDecryptService) {
val logger: Logger = Logger(this.getClass())
def createDbConfigCustomized (dbConfigProvider: DatabaseConfigProvider) : DatabaseConfig[JdbcProfile] = {
val user: String = encryptDecryptService.decrypt(config.get[String]("slick.dbs.default.db.user"))
val pass: String = encryptDecryptService.decrypt(config.get[String]("slick.dbs.default.db.password"))
val dbConfigOwn = dbConfigProvider.get[JdbcProfile]
val decryptedConfig = dbConfigOwn.config
.withValue("db.user", ConfigValueFactory.fromAnyRef(user))
.withValue("db.password", ConfigValueFactory.fromAnyRef(pass))
DatabaseConfig.forConfig[JdbcProfile]("", decryptedConfig)
}
}
and this:
class Reservations #Inject()(val dbConfigProvider: DatabaseConfigProvider,
config: Configuration,
customizedSlickConfig: CustomizedSlickConfig)
extends HasDatabaseConfigProviderTalachitas[JdbcProfile] {
val logger: Logger = Logger(this.getClass())
this.dbConfig = customizedSlickConfig.createDbConfigCustomized(dbConfigProvider)
needed to override completely that default slick using some https://github.com/google/tink library for encryption