docker daemon预置iptables剖析

没有安装docker的机器：iptables为空，且每个链路的默认policy均为ACCEPT

[root@~]# iptables-save[root@ ~]# iptables -t raw -nvL
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain OUTPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destination[root@~]# iptables -t mangle -nvL
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain INPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain FORWARD (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain OUTPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destination[root@~]# iptables -t filter -nvL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain FORWARD (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain OUTPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destination[root@~]# iptables -t nat -nvL
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain INPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain OUTPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destination

安装了docker后的机器

[root@~]# iptables-save
# Generated by iptables-save v1.8.4 on Mon Jul 10 18:12:12 2023
*raw
:PREROUTING ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
COMMIT
# Completed on Mon Jul 10 18:12:12 2023
# Generated by iptables-save v1.8.4 on Mon Jul 10 18:12:12 2023
*mangle
:PREROUTING ACCEPT [0:0]
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
COMMIT
# Completed on Mon Jul 10 18:12:12 2023
# Generated by iptables-save v1.8.4 on Mon Jul 10 18:12:12 2023
*filter
:INPUT ACCEPT [0:0]
:FORWARD DROP [0:0]
:OUTPUT ACCEPT [0:0]
:DOCKER - [0:0]
:DOCKER-ISOLATION-STAGE-1 - [0:0]
:DOCKER-ISOLATION-STAGE-2 - [0:0]
:DOCKER-USER - [0:0]
-A FORWARD -j DOCKER-USER
-A FORWARD -j DOCKER-ISOLATION-STAGE-1
-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -o docker0 -j DOCKER
-A FORWARD -i docker0 ! -o docker0 -j ACCEPT
-A FORWARD -i docker0 -o docker0 -j ACCEPT
-A DOCKER-ISOLATION-STAGE-1 -i docker0 ! -o docker0 -j DOCKER-ISOLATION-STAGE-2
-A DOCKER-ISOLATION-STAGE-1 -j RETURN
-A DOCKER-ISOLATION-STAGE-2 -o docker0 -j DROP
-A DOCKER-ISOLATION-STAGE-2 -j RETURN
-A DOCKER-USER -j RETURN
COMMIT
# Completed on Mon Jul 10 18:12:12 2023
# Generated by iptables-save v1.8.4 on Mon Jul 10 18:12:12 2023
*nat
:PREROUTING ACCEPT [0:0]
:INPUT ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:DOCKER - [0:0]
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE
-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-A DOCKER -i docker0 -j RETURN
COMMIT
# Completed on Mon Jul 10 18:12:12 2023[root@~]# iptables -t raw -nvL
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain OUTPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destination[root@~]# iptables -t filter -nvL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain FORWARD (policy DROP 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destination0     0 DOCKER-USER  all  --  *      *       0.0.0.0/0            0.0.0.0/00     0 DOCKER-ISOLATION-STAGE-1  all  --  *      *       0.0.0.0/0            0.0.0.0/00     0 ACCEPT     all  --  *      docker0  0.0.0.0/0            0.0.0.0/0            ctstate RELATED,ESTABLISHED0     0 DOCKER     all  --  *      docker0  0.0.0.0/0            0.0.0.0/00     0 ACCEPT     all  --  docker0 !docker0  0.0.0.0/0            0.0.0.0/00     0 ACCEPT     all  --  docker0 docker0  0.0.0.0/0            0.0.0.0/0Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain DOCKER (1 references)pkts bytes target     prot opt in     out     source               destinationChain DOCKER-ISOLATION-STAGE-1 (1 references)pkts bytes target     prot opt in     out     source               destination0     0 DOCKER-ISOLATION-STAGE-2  all  --  docker0 !docker0  0.0.0.0/0            0.0.0.0/00     0 RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0Chain DOCKER-ISOLATION-STAGE-2 (1 references)pkts bytes target     prot opt in     out     source               destination0     0 DROP       all  --  *      docker0  0.0.0.0/0            0.0.0.0/00     0 RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0Chain DOCKER-USER (1 references)pkts bytes target     prot opt in     out     source               destination0     0 RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0[root@~]# iptables -t mangle -nvL
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain INPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain FORWARD (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain OUTPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destination[root@~]# iptables -t nat -nvL
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destination5  1121 DOCKER     all  --  *      *       0.0.0.0/0            0.0.0.0/0            ADDRTYPE match dst-type LOCALChain INPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destination0     0 MASQUERADE  all  --  *      !docker0  172.17.0.0/16        0.0.0.0/0Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destination3   168 DOCKER     all  --  *      *       0.0.0.0/0           !127.0.0.0/8          ADDRTYPE match dst-type LOCALChain DOCKER (2 references)pkts bytes target     prot opt in     out     source               destination0     0 RETURN     all  --  docker0 *       0.0.0.0/0            0.0.0.0/0

docker基础的iptables网络要实现以下目标

1、docker网络与宿主机网络分开：

在机器上下发一条iptables，实际只可以拦截宿主机其他进程，拦截不了docker进程

iptables -P INPUT DROP

所谓的 Docker 网络的隔离性只在 INPUT 链，OUTPUT 链中体现。修改 PREROUTING 链，FORWARD 链，POSTROUTING 链都会影响到 Docker 容器的网络环境。要通过 DOCKER 链控制 Docker 容器的访问权限，需要先删除 -A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 这条规则, 因为这条规则的不确定性太大。然后单纯通过 DOCKER 链来控制 Docker 容器的网络访问权限.
容器的请求没有经过宿主机的 INPUT 链，所以在宿主机的 INPUT 链上做规则是没法限制容器的网络访问的. 需要限制容器网络访问应该对 DOCKER 链动手脚。

2、docker网络里不同docker桥之间网络不互通

3、将FORWARD链的默认规则改为drop，原因如下：

当 docker 启动时，它会启用net.ipv4.ip_forward而不将 iptablesFORWARD链默认策略更改为DROP。这意味着与 docker 主机位于同一网络上的另一台计算机可以将路由添加到其路由表中，并直接寻址该 docker 主机上运行的任何容器。
issue：https://github.com/moby/moby/issues/14041
要转发流量需要net.ipv4.ip_forward，而net.ipv4.ip_forward默认是关的
net.ipv4.ip_forward 只要在docker daemon 启动之前手动把这个参数设置为1，docker daemon 默认就认为有人用iptables的filter表的forward链了，就不会更改forward链的默认策略了。

docker iptables链剖析

docker的几条链路：

Docker的DOCKER链：仅处理从宿主机到docker0的IP数据包。
Docker的DOCKER-ISOLATION链：为了隔离在不同的bridge网络之间的容器，Docker提供了两个DOCKER-ISOLATION阶段实现。
- DOCKER-ISOLATION-STAGE-1链过滤源地址是bridge网络（默认docker0）的IP数据包，匹配的IP数据包再进入DOCKER-ISOLATION-STAGE-2链处理，不匹配就返回到父链FORWARD。
- 在DOCKER-ISOLATION-STAGE-2链中，进一步处理目的地址是bridge网络的IP数据包，匹配的IP数据包表示该IP数据包是从一个bridge网络的网桥发出，
  到另一个bridge网络的网桥，这样的IP数据包来自其他bridge网络，将被直接DROP；不匹配的IP数据包就返回到父链FORWARD继续进行后续处理。
Docker的DOCKER-USER链

用于用户可以自定义添加自己对容器的处理规则

注意：

Docker启动时，会加载DOCKER链和DOCKER-ISOLATION（现在是DOCKER-ISOLATION-STAGE-1）链中的过滤规则，并使之生效，绝对禁止修改这里的过滤规则。
如果用户要补充Docker的过滤规则，强烈建议追加到DOCKER-USER链。DOCKER-USER链中的过滤规则，将先于Docker默认创建的规则被加载，从而能够覆盖Docker在DOCKER链和DOCKER-ISOLATION链中的默认过滤规则。例如，Docker启动后，默认任何外部source IP都被允许转发，从而能够从该source IP连接到宿主机上的任何Docker容器实例。如果只允许一个指定的IP访问容器实例，可以插入路由规则到DOCKER-USER链中，从而能够在DOCKER链之前被加载。

iptables规则分析：

1、DOCKER链：对入包和出包都拦截到自定义Docker链路

-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A DOCKER -i docker0 -j RETURN

2、DOCKER-ISOLATION链：

默认只有docker0，此时可能不好理解，对于源是docker0，目的不是docker0，且目的是docker0的
流量进行丢弃，看起来好像矛盾，因为对于入向是docker0，出向不是docker0的流量跳转到
DOCKER-ISOLATION-STAGE-2时，必定不会再匹配上了，但是其实是因为默认值只有docker0，
默认情况下DOCKER-ISOLATION-STAGE-2的drop不会命中，因为只有一个docker网桥-A DOCKER-ISOLATION-STAGE-1 -i docker0 ! -o docker0 -j DOCKER-ISOLATION-STAGE-2
-A DOCKER-ISOLATION-STAGE-1 -j RETURN
-A DOCKER-ISOLATION-STAGE-2 -o docker0 -j DROP
-A DOCKER-ISOLATION-STAGE-2 -j RETURN当再添加一个网桥的时候，docker network create -d bridge docker1，此时就变成了：
-A DOCKER-ISOLATION-STAGE-1 -i br-fc313d59a28d ! -o br-fc313d59a28d -j DOCKER-ISOLATION-STAGE-2
-A DOCKER-ISOLATION-STAGE-1 -i docker0 ! -o docker0 -j DOCKER-ISOLATION-STAGE-2
-A DOCKER-ISOLATION-STAGE-1 -j RETURN
-A DOCKER-ISOLATION-STAGE-2 -o br-fc313d59a28d -j DROP
-A DOCKER-ISOLATION-STAGE-2 -o docker0 -j DROP
-A DOCKER-ISOLATION-STAGE-2 -j RETURN这时候就解释通了，配合起来就是：
源是br-fc313d59a28d，目的不是br-fc313d59a28d，但目的是docker0的会被drop，
源是docker0，目的不是docker0，但目的是br-fc313d59a28d的会被drop。
也就是说-A DOCKER-ISOLATION-STAGE-2 -o docker0 -j DROP是为了去拦截非docker0的
那些网桥来访问docker0，而-A DOCKER-ISOLATION-STAGE-2 -o br-fc313d59a28d -j DROP
是为了去拦截非br-fc313d59a28d的那些网桥来访问br-fc313d59a28d

DOCKER-USER链：存放用户自己的链

隔离规则再剖析：

再创建一个docker网络桥：可以看到在隔离的规则除了原来的docker0，同时也为docker0对应的网桥br-fc313d59a28d也多了限制规则

docker network create -d bridge docker1[root@10-234-37-2 ~]# iptables -t filter -nvL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain FORWARD (policy DROP 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destination0     0 DOCKER-USER  all  --  *      *       0.0.0.0/0            0.0.0.0/00     0 DOCKER-ISOLATION-STAGE-1  all  --  *      *       0.0.0.0/0            0.0.0.0/00     0 ACCEPT     all  --  *      br-fc313d59a28d  0.0.0.0/0            0.0.0.0/0            ctstate RELATED,ESTABLISHED0     0 DOCKER     all  --  *      br-fc313d59a28d  0.0.0.0/0            0.0.0.0/00     0 ACCEPT     all  --  br-fc313d59a28d !br-fc313d59a28d  0.0.0.0/0            0.0.0.0/00     0 ACCEPT     all  --  br-fc313d59a28d br-fc313d59a28d  0.0.0.0/0            0.0.0.0/00     0 ACCEPT     all  --  *      docker0  0.0.0.0/0            0.0.0.0/0            ctstate RELATED,ESTABLISHED0     0 DOCKER     all  --  *      docker0  0.0.0.0/0            0.0.0.0/00     0 ACCEPT     all  --  docker0 !docker0  0.0.0.0/0            0.0.0.0/00     0 ACCEPT     all  --  docker0 docker0  0.0.0.0/0            0.0.0.0/0Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)pkts bytes target     prot opt in     out     source               destinationChain DOCKER (2 references)pkts bytes target     prot opt in     out     source               destinationChain DOCKER-ISOLATION-STAGE-1 (1 references)pkts bytes target     prot opt in     out     source               destination0     0 DOCKER-ISOLATION-STAGE-2  all  --  br-fc313d59a28d !br-fc313d59a28d  0.0.0.0/0            0.0.0.0/00     0 DOCKER-ISOLATION-STAGE-2  all  --  docker0 !docker0  0.0.0.0/0            0.0.0.0/00     0 RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0Chain DOCKER-ISOLATION-STAGE-2 (2 references)pkts bytes target     prot opt in     out     source               destination0     0 DROP       all  --  *      br-fc313d59a28d  0.0.0.0/0            0.0.0.0/00     0 DROP       all  --  *      docker0  0.0.0.0/0            0.0.0.0/00     0 RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0Chain DOCKER-USER (1 references)pkts bytes target     prot opt in     out     source               destination0     0 RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0