Apache Flink Stateful Functions deployment

Overview:

Flink jobs deployment types:

  • in Application Mode
  • in a Per-Job Mode
  • in Session Mode
  • the cluster lifecycle and resource isolation guarantees
  • whether the application’s main() method is executed on the client or on the cluster.

Application Mode

Per-Job Mode

Session Mode

Stateful Functions building and deployment

Building

FROM flink-statefun:2.2.0

RUN mkdir -p /opt/statefun/modules/statefun-example
RUN mkdir -p /opt/statefun/modules/remote

COPY target/statefun-example*jar /opt/statefun/modules/statefun-example/
COPY module.yaml /opt/statefun/modules/remote/module.yaml
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>statefun-flink-distribution</artifactId>
<version>2.2.0</version>
</dependency>
./bin/flink run -c org.apache.flink.statefun.flink.core.StatefulFunctionsJob ./statefun-example.jar
classloader.parent-first-patterns.additional: org.apache.flink.statefun;org.apache.kafka;com.google.protobuf

Deployment

Remote Functions

Co-located Functions

Embedded Functions

Flink Docker deployment

command:            
- "/usr/local/bin/bash"

args: ['-c',
'export JOB_FILENAME=$(basename $DOWNLOAD_URL);
echo "DOWNLOAD_URL: $DOWNLOAD_URL FILE: $JOB_FILENAME JOBMANAGER: $JOBMANAGER";
apk add --no-cache jq curl;
curl -s $DOWNLOAD_URL -o $JOB_FILENAME;
curl -s -X POST -H "Expect:" -F "jarfile=@$JOB_FILENAME" $JOBMANAGER:8081/jars/upload;
while true; do date; export JAR_ID=$(curl -s $JOBMANAGER:8081/jars | jq -r ".files[].id");
if [ -z $JAR_ID ];
then
echo "No uploaded jar detected";
else
echo "Found jar $JAR_ID";
export SUBMIT_MSG=$(curl -s -X POST -H "Expect:" $JOBMANAGER:8081/jars/$JAR_ID/run?program-args=--kafka_host%20kafka-kafka-1.kafka-svc.default.svc.cluster.local:9093 | jq -r ".errors"); echo "RESPONSE: $SUBMIT_MSG";
if [ $SUBMIT_MSG == "null" ];
then
echo "SUBMITTED JOB!";
exit 0;
else
echo "Failed to submit job: $SUBMIT_MSG";
fi;
fi;
echo "=====================";
sleep 5; done;']
version: "2.1"
services:
zookeeper:
image: wurstmeister/zookeeper
ports:
- "2181:2181"
kafka-broker:
image: wurstmeister/kafka:2.12-2.0.1
ports:
- "9092:9092"
environment:
HOSTNAME_COMMAND: "route -n | awk '/UG[ \t]/{print $$2}'"
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
depends_on:
- zookeeper
volumes:
- /var/run/docker.sock:/var/run/docker.sock
master:
build:
context: .
expose:
- "6123"
ports:
- "8081:8081"
environment:
- ROLE=master
- MASTER_HOST=master
volumes:
- ./checkpoint-dir:/checkpoint-dir
db:
image: postgres
environment:
- POSTGRES_DB=postgres
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
ports:
- "5432:5432"
worker:
build:
context: .
expose:
- "6121"
- "6122"
depends_on:
- master
- kafka-broker
links:
- "master:master"
- "kafka-broker:kafka-broker"
environment:
- ROLE=worker
- MASTER_HOST=master
volumes:
- ./checkpoint-dir:/checkpoint-dir
python-worker:
build:
context: ./greeter
expose:
- "8000"
event-generator:
build:
context: generator
dockerfile: Dockerfile
depends_on:
- kafka-broker

Summary:

URLs:

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store