https://github.com/azkaban/azkaban/tags
https://codeload.github.com/azkaban/azkaban/tar.gz/refs/tags/3.84.4
Azkaban是一套简单的任务调度服务，整体包括三部分webserver、dbserver、executorserver。Azkaban 为 LinkedIn 开源的分布式工作流调度框架，项目起源于解决 Hadoop Jop 依赖管理问题，但实际已经是一个通用的工作流调度框架，适用于不同业务场景。XXL-JOB 是一个国内轻量级分布式任务调度平台，其核心设计目标是开发迅速、学习简单、轻量级、易扩展
[root@vm ~]# yum -y install git java gcc gcc-c++* make
[root@vm ~]# curl --silent --location https://rpm.nodesource.com/setup_8.x | bash -
[root@vm ~]# yum install -y nodejs
[root@vm ~]# tar xf azkaban-3.84.4.tar.gz
[root@vm ~]# cd azkaban-3.84.4
[root@vm azkaban-3.84.4]# ./gradlew cleanERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.[root@vm azkaban-3.84.4]# export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.362.b08-1.el7_9.x86_64/jre/[root@vm azkaban-3.84.4]# vim +53 build.gradle
...
allprojects {apply plugin: 'jacoco'repositories {mavenCentral()mavenLocal()maven { url 'https://maven.aliyun.com/repository/google' }maven { url 'https://maven.aliyun.com/repository/jcenter' }maven { url 'http://maven.aliyun.com/nexus/content/groups/public' }maven { url 'http://download.flutter.io'}}
}[root@vm azkaban-3.84.4]# ./gradlew clean
Downloading https://services.gradle.org/distributions/gradle-4.6-all.zip
...................
BUILD SUCCESSFUL in 1m 27s
20 actionable tasks: 20 up-to-date[root@vm azkaban-3.84.4]# vi +33 azkaban-web-server/build.gradle
node {
...download = false
[root@vm azkaban-3.84.4]# ./gradlew build -x test[root@vm azkaban-3.84.4]# ls azkaban-db/build/distributions/azkaban-db-0.1.0-SNAPSHOT.tar.gz
azkaban-db/build/distributions/azkaban-db-0.1.0-SNAPSHOT.tar.gz
[root@vm azkaban-3.84.4]# ls azkaban-exec-server/build/distributions/azkaban-exec-server-0.1.0-SNAPSHOT.tar.gz
azkaban-exec-server/build/distributions/azkaban-exec-server-0.1.0-SNAPSHOT.tar.gz
[root@vm azkaban-3.84.4]# ls azkaban-web-server/build/distributions/azkaban-web-server-0.1.0-SNAPSHOT.tar.gz
azkaban-web-server/build/distributions/azkaban-web-server-0.1.0-SNAPSHOT.tar.gzAzkaban-Solo 方式运行
[root@vm azkaban-3.84.4]# mkdir /opt/azkaban
[root@vm azkaban-3.84.4]# cd azkaban-solo-server/build/distributions/
[root@vm distributions]# tar xf azkaban-solo-server-0.1.0-SNAPSHOT.tar.gz
[root@vm distributions]# mv azkaban-solo-server-0.1.0-SNAPSHOT /opt/azkaban/azkaban-solo-server
[root@vm distributions]# cd /opt/azkaban/azkaban-solo-server
[root@vm azkaban-solo-server]# vim conf/azkaban.properties
default.timezone.id=Asia/Shanghai
[root@vm azkaban-solo-server]# bin/start-solo.sh[root@vm azkaban-solo-server]# cat conf/azkaban-users.xml #登录名
<azkaban-users><user groups="azkaban" password="azkaban" roles="admin" username="azkaban"/>
[root@vm azkaban-solo-server]# ./bin/shutdown-solo.sh
生成job任务zip包
[root@vm ~]# vim command.job
#command.job
type=command
command=echo 'hello' [root@vm ~]# zip -r command.zip command.job
http://10.1.0.18:8081 azkaban/azkaban
Create Project -->Upload Project Files -->command.zip -->run job --Execute --continue
Schedule Flow Options 计划任务执行
http://10.1.0.18:8081/index 查看
two-server 方式运行
[root@vm azkaban-3.84.4]# tar xf azkaban-db/build/distributions/azkaban-db-0.1.0-SNAPSHOT.tar.gz -C /opt/azkaban/
[root@vm azkaban-3.84.4]# tar xf azkaban-exec-server/build/distributions/azkaban-exec-server-0.1.0-SNAPSHOT.tar.gz -C /opt/azkaban/
[root@vm azkaban-3.84.4]# tar xf azkaban-web-server/build/distributions/azkaban-web-server-0.1.0-SNAPSHOT.tar.gz -C /opt/azkaban/
[root@vm azkaban-3.84.4]# tar xf azkaban-solo-server/build/distributions/azkaban-solo-server-0.1.0-SNAPSHOT.tar.gz -C /opt/azkaban/[root@vm azkaban-3.84.4]# cd /opt/azkaban/
[root@vm azkaban]# mv azkaban-db-0.1.0-SNAPSHOT db
[root@vm azkaban]# mv azkaban-exec-server-0.1.0-SNAPSHOT exec-server
[root@vm azkaban]# mv azkaban-web-server-0.1.0-SNAPSHOT web-server
[root@vm azkaban]# mv azkaban-solo-server-0.1.0-SNAPSHOT solo-server
[root@vm azkaban]# ls
db exec-server solo-server web-server
[root@vm azkaban]# docker run -d -p3306:3306 --name mysql -e MYSQL_ROOT_PASSWORD=123456 mysql:5.7.10
[root@vm ~]# docker cp /opt/azkaban/db/create-all-sql-0.1.0-SNAPSHOT.sql mysql:/root/[root@vm azkaban]# docker exec -it mysql bash
root@e00138f5b17a:/# mysql -uroot -p123456
mysql> create database azkaban;
mysql> use azkaban;
Database changed
mysql> source /root/create-all-sql-0.1.0-SNAPSHOT.sql
mysql> create user 'azkaban'@'%' identified by 'azkaban';
mysql> grant select,insert,update,delete on azkaban.* to 'azkaban'@'%' with grant option;
mysql> flush privileges;
mysql> exit
配置exec-server
[root@vm azkaban]# cd exec-server/
[root@vm exec-server]# vim conf/azkaban.propertiesdefault.timezone.id=Asia/Shanghai
azkaban.webserver.url=http://192.168.56.10:8081database.type=mysql
mysql.port=3306
mysql.host=192.168.56.10
mysql.database=azkaban
mysql.user=azkaban
mysql.password=azkaban
mysql.numconnections=100
# Azkaban Executor settings
executor.maxThreads=50
executor.flow.threads=30
executor.port=12321
executor.metric.reports=true
executor.metric.milisecinterval.default=60000[root@vm exec-server]# vim plugins/jobtypes/commonprivate.properties
execute.as.user=false
azkaban.native.lib=false
memCheck.enabled=false[root@vm exec-server]# vim conf/log4j.properties
log4j.appender.server.File=logs/azkaban-execserver.log
[root@vm exec-server]# bin/start-exec.sh
[root@vm exec-server]# ss -antup |grep 12321
tcp LISTEN 0 50 [::]:12321
[root@vm exec-server]# curl -G 192.168.56.10:12321/executor?action=activate
{"status":"success"}[root@vm ~]# docker exec -it mysql mysql -uroot -p123456 -e "select * from azkaban.executors;"
+----+------+-------+--------+
| id | host | port | active |
+----+------+-------+--------+
| 1 | vm | 12321 | 1 |
+----+------+-------+--------+配置web-server
[root@vm exec-server]# cd ../web-server/
[root@vm web-server]# vim conf/azkaban.properties
default.timezone.id=Asia/Shanghai
database.type=mysql
mysql.port=3306
mysql.host=192.168.56.10
mysql.database=azkaban
mysql.user=azkaban
mysql.password=azkaban
mysql.numconnections=100
#注释azkaban.executorselector.filters=StaticRemainingFlowSize,MinimumFreeMemory,CpuStatus
#可能会因为exec-server资源不足，导致任务不执行，一直preparing
[root@vm web-server]# vim conf/azkaban-users.xml
...<user username="admin" password="admin" roles="admin,metrics" />
[root@vm web-server]# vim conf/log4j.properties
log4j.appender.server.File= #也可以不改[root@vm web-server]# bin/start-web.sh
[root@vm web-server]# jps
2082 AzkabanExecutorServer
2261 AzkabanWebServer
2315 Jpshttp://192.168.56.10:8081/index #Create Project -->Upload Project Files同上面solo模式执行
[root@vm web-server]# vim conf/azkaban.properties#在每次分发job时，先过滤出满足条件的executor，如最小剩余内存>6G cpu占用率< 95%,参考官网,
#低配置服务器注释掉下面的filter
azkaban.executorselector.filters=StaticRemainingFlowSize,MinimumFreeMemory,CpuStatus# 然后再做比较筛选由于是虚拟机，不需要过滤，只需要比较即可
azkaban.executorselector.comparator.NumberOfAssignedFlowComparator=1 # 某个任务是否指定了executor id
azkaban.executorselector.comparator.Memory=1 # 是否比较内存
azkaban.executorselector.comparator.LastDispatched=1 # 是否最后一次被分发
azkaban.executorselector.comparator.CpuUsage=1 # 是否比较CPU
multiple-executor方式运行
单节点webServer executor
web-server修改配置文件
[root@vm web-server]# vim conf/azkaban.properties
#启用multiple-executor模式
azkaban.use.multiple.executors=true
[root@vm web-server]# bin/start-web.sh[root@vm azkaban]# cp -r exec-server/ exec-server2
[root@vm azkaban]# vim exec-server2/conf/azkaban.properties
executor.port=12322[root@vm exec-server]# cd exec-server;bin/start-exec.sh
[root@vm exec-server]# cd ../exec-server2/ ;bin/start-exec.sh
[root@vm exec-server2]# curl -G 192.168.56.10:12321/executor?action=activate
[root@vm exec-server2]# curl -G 192.168.56.10:12322/executor?action=activatehttp://192.168.56.10:8081/index #访问测试
分布式webServer executor
192.168.56.10: webServer mysql
10.2.0.11: executor
10.2.0.12: executor> vim /etc/hosts # webServer
10.2.0.11 master-1
10.2.0.12 master-2> vim /etc/hosts # executor
192.168.56.10 azkaban
mysql> use azkaban;
mysql> insert into executors(host,port,active) values("master-1",12321,1);
mysql> insert into executors(host,port,active) values("master-2",12321,1);
[root@os1 exec-server]# vi conf/azkaban.properties
...
default.timezone.id=Asia/Shanghaiazkaban.webserver.url=http://azkaban:8081database.type=mysql
mysql.port=3306
mysql.host=192.168.56.10
mysql.database=azkaban
mysql.user=azkaban
mysql.password=azkaban
mysql.numconnections=100
...
executor.port=12321
executor.metric.reports=true
executor.metric.milisecinterval.default=60000[root@os1 exec-server]# bin/start-exec.sh
[root@vm azkaban-web-server-0.1.0-SNAPSHOT]# vim conf/azkaban.properties
database.type=mysql
mysql.port=3306
mysql.host=192.168.56.10
mysql.database=azkaban
mysql.user=azkaban
mysql.password=azkaban
mysql.numconnections=100
#Multiple Executor
azkaban.use.multiple.executors=true
#azkaban.executorselector.filters=StaticRemainingFlowSize,MinimumFreeMemory,CpuStatus
...
[root@vm azkaban-web-server-0.1.0-SNAPSHOT]# vim conf/azkaban-users.xml<user username="admin" password="admin" roles="admin,metrics" />[root@vm azkaban-web-server-0.1.0-SNAPSHOT]# bin/start-web.sh[root@vm azkaban-web-server-0.1.0-SNAPSHOT]# curl -G 10.2.0.11:12321/executor?action=activate
{"status":"success"} #不做这个操作还是无法执行任务
[root@vm azkaban-web-server-0.1.0-SNAPSHOT]# curl -G 10.2.0.12:12321/executor?{"status":"success"}http://localhost:8081/ 登录 操作通solo。查看history里的执行记录
成功的日志里显示executor的执行信息17:33:17 CST command INFO - Assigned executor : os1:12321
