0%

OpenvSwitch source code(2)

Posted on Views: Disqus:
這篇文章中，我們決定透過閱讀原始碼的方式，來瞭解 OpenvSwitch 操作上最常使用的指令，也就是 add-port 這個指令每次運行時，整個系統到底怎麼運行的。藉由閱讀原始碼的方式來釐清整個 OpenvSwitch 的架構，從 User-space 的程序到 Kerenel Space 的 Module, 這中間到底是怎麼處理的。

Preface

當使用 ovs-vsctl add-port br0 eth1的時候，實際上會做什麼事情，這邊分成兩個層面來看

  • User space
  • Kernel space

而這之中則會透過 generic netlink來溝通

User space

這部分還有點卡住，對 ovsdb 有更瞭解後再補充
bridge_reconfigure
bridge_refresh_ofp_port
bridge_refresh_one_ofp_port
iface_create
iface_do_create
ofproto_port_add

VLOG_INFO

1
2
3
4
5
6
7
8
static int
port_add(struct ofproto *ofproto_, struct netdev *netdev)
{
...
	error = dpif_port_add(ofproto->backer->dpif, netdev, &port_no);

...
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
int
dpif_port_add(struct dpif *dpif, struct netdev *netdev, odp_port_t *port_nop)
{
    const char *netdev_name = netdev_get_name(netdev);
    odp_port_t port_no = ODPP_NONE;
    int error;

    COVERAGE_INC(dpif_port_add);

    if (port_nop) {
        port_no = *port_nop;
    }

    error = dpif->dpif_class->port_add(dpif, netdev, &port_no);
    if (!error) {
        VLOG_DBG_RL(&dpmsg_rl, "%s: added %s as port %"PRIu32,
                    dpif_name(dpif), netdev_name, port_no);
    } else {
        VLOG_WARN_RL(&error_rl, "%s: failed to add %s as port: %s",
                     dpif_name(dpif), netdev_name, ovs_strerror(error));
        port_no = ODPP_NONE;
    }
    if (port_nop) {
        *port_nop = port_no;
    }
    return error;
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
static void
dp_initialize(void)
{
    static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER;

    if (ovsthread_once_start(&once)) {
        int i;

        for (i = 0; i < ARRAY_SIZE(base_dpif_classes); i++) {
            dp_register_provider(base_dpif_classes[i]);
        }
        ovsthread_once_done(&once);
    }
}
  • 這邊會去註冊每個 base_dipf_classes.
1
2
3
4
5
6
7
8
9
10
11
12
13
struct dpif_class {
....
    /* Adds 'netdev' as a new port in 'dpif'.  If '*port_no' is not
     * UINT32_MAX, attempts to use that as the port's port number.
     *
     * If port is successfully added, sets '*port_no' to the new port's
     * port number.  Returns EBUSY if caller attempted to choose a port
     * number, and it was in use. */
    int (*port_add)(struct dpif *dpif, struct netdev *netdev,
                    odp_port_t *port_no);
...

}
  • dpif_class是一個base class,裡面存放的都是function pointer.
1
2
3
4
5
6
static const struct dpif_class *base_dpif_classes[] = {
#ifdef LINUX_DATAPATH
    &dpif_linux_class,
#endif
    &dpif_netdev_class,
};
  • 這邊會根據type去實例化不同類型的 dpif_class,這邊我們關心的是 dpif_linux_class.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
const struct dpif_class dpif_linux_class = {
    "system",
    dpif_linux_enumerate,
    NULL,
    dpif_linux_open,
    dpif_linux_close,
    dpif_linux_destroy,
    NULL,                       /* run */
    NULL,                       /* wait */
    dpif_linux_get_stats,
    dpif_linux_port_add,
    dpif_linux_port_del,
    dpif_linux_port_query_by_number,
    dpif_linux_port_query_by_name,
    dpif_linux_get_max_ports,
    dpif_linux_port_get_pid,
    dpif_linux_port_dump_start,
    dpif_linux_port_dump_next,
    dpif_linux_port_dump_done,
    dpif_linux_port_poll,
    dpif_linux_port_poll_wait,
    dpif_linux_flow_get,
    dpif_linux_flow_put,
    dpif_linux_flow_del,
    dpif_linux_flow_flush,
    dpif_linux_flow_dump_start,
    dpif_linux_flow_dump_next,
    dpif_linux_flow_dump_done,
    dpif_linux_execute,
    dpif_linux_operate,
    dpif_linux_recv_set,
    dpif_linux_queue_to_priority,
    dpif_linux_recv,
    dpif_linux_recv_wait,
    dpif_linux_recv_purge,
};
  • 這邊定義了 dpif_class的一些操作function.
  • 我們關心的 port_add 實際上對應的是 dpif_linux_port_add
1
2
3
4
5
6
7
8
9
10
11
12
13
static int
dpif_linux_port_add(struct dpif *dpif_, struct netdev *netdev,
                    odp_port_t *port_nop)
{
    struct dpif_linux *dpif = dpif_linux_cast(dpif_);
    int error;

    ovs_mutex_lock(&dpif->upcall_lock);
    error = dpif_linux_port_add__(dpif_, netdev, port_nop);
    ovs_mutex_unlock(&dpif->upcall_lock);

    return error;
}
  • 這邊會呼叫 dpif_linux_port_add__去創立vport.
1
2
3
4
5
6
7
8
9
10
11
12
static int
dpif_linux_port_add__(struct dpif *dpif_, struct netdev *netdev,
                      odp_port_t *port_nop)
{
....
    request.cmd = OVS_VPORT_CMD_NEW;
    request.dp_ifindex = dpif->dp_ifindex;
    request.type = netdev_to_ovs_vport_type(netdev);
...
	 error = dpif_linux_vport_transact(&request, &reply, &buf);
...
}
  • 這邊會設定request的cmd為 OVS_VPORT_CMD_NEW
  • 使用 dpif_linux_vport_transact 把這個request透過netlink的方式送到kernel端，kernel端會再根據這個request的cmd來執行特定的function.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
static int dp_register_genl(void)
{
        int n_registered;
        int err;
        int i;

        n_registered = 0;
        for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
                const struct genl_family_and_ops *f = &dp_genl_families[i];

                err = genl_register_family_with_ops(f->family, f->ops,
                                                    f->n_ops);
                if (err)
                        goto error;
                n_registered++;

                if (f->group) {
                        err = genl_register_mc_group(f->family, f->group);
                        if (err)
                                goto error;
                }
        }

        return 0;

error:
        dp_unregister_genl(n_registered);
        return err;
}
  • 當 datapath kernel module被載入的時候，會執行對應的 init,裡面會執行 dp_register_genl.
  • 這邊會呼叫所有的 dp_genl_families 來註冊 generic netlink相關的function.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
static const struct genl_family_and_ops dp_genl_families[] = {
        { &dp_datapath_genl_family,
          dp_datapath_genl_ops, ARRAY_SIZE(dp_datapath_genl_ops),
          &ovs_dp_datapath_multicast_group },
        { &dp_vport_genl_family,
          dp_vport_genl_ops, ARRAY_SIZE(dp_vport_genl_ops),
          &ovs_dp_vport_multicast_group },
        { &dp_flow_genl_family,
          dp_flow_genl_ops, ARRAY_SIZE(dp_flow_genl_ops),
          &ovs_dp_flow_multicast_group },
        { &dp_packet_genl_family,
          dp_packet_genl_ops, ARRAY_SIZE(dp_packet_genl_ops),
          NULL },
};
  • 這邊定義了所有 gerneric netlink相關type的結構成員 (dp,vport,flow,packet)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
static struct genl_ops dp_vport_genl_ops[] = {
        { .cmd = OVS_VPORT_CMD_NEW,
          .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
          .policy = vport_policy,
          .doit = ovs_vport_cmd_new
        },
        { .cmd = OVS_VPORT_CMD_DEL,
          .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
          .policy = vport_policy,
          .doit = ovs_vport_cmd_del
        },
        { .cmd = OVS_VPORT_CMD_GET,
          .flags = 0,               /* OK for unprivileged users. */
          .policy = vport_policy,
          .doit = ovs_vport_cmd_get,
          .dumpit = ovs_vport_cmd_dump
        },
        { .cmd = OVS_VPORT_CMD_SET,
          .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
          .policy = vport_policy,
          .doit = ovs_vport_cmd_set,
        },
};
  • 這邊定義了 gerneric netlink 相關的vport operation.
  • 當 cmd 是 OVS_VPORT_CMD_NEW的時候，就會執行對應的 function handler ovs_vport_cmd_new

Kernel space

1
2
3
4
5
6
7
8
9
10
11
12
13
static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
{
...
        parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
        parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
        parms.options = a[OVS_VPORT_ATTR_OPTIONS];
        parms.dp = dp;
        parms.port_no = port_no;
        parms.upcall_portid = nla_get_u32(a[OVS_VPORT_ATTR_UPCALL_PID]);

        vport = new_vport(&parms);
...
}
  • struct vport_parms parms 用來記錄一些vport的資訊
  • 填寫相關訊息後，就呼叫 new_vport來創立 vport
1
2
3
4
5
6
7
8
9
10
11
12
13
static struct vport *new_vport(const struct vport_parms *parms)
{
        struct vport *vport;

        vport = ovs_vport_add(parms);
        if (!IS_ERR(vport)) {
                struct datapath *dp = parms->dp;
                struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);

                hlist_add_head_rcu(&vport->dp_hash_node, head);
        }
        return vport;
}
  • 這邊會呼叫 *ovs_vport_add 來創立該vport
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
struct vport *ovs_vport_add(const struct vport_parms *parms)
{
        struct vport *vport;
        int err = 0;
        int i;

        for (i = 0; i < ARRAY_SIZE(vport_ops_list); i++) {
                if (vport_ops_list[i]->type == parms->type) {
                        struct hlist_head *bucket;

                        vport = vport_ops_list[i]->create(parms);
                        if (IS_ERR(vport)) {
                                err = PTR_ERR(vport);
                                goto out;
                        }

                        bucket = hash_bucket(ovs_dp_get_net(vport->dp),
                                             vport->ops->get_name(vport));
                        hlist_add_head_rcu(&vport->hash_node, bucket);
                        return vport;
                }
        }

        err = -EAFNOSUPPORT;

out:
        return ERR_PTR(err);
}
  • 掃過所有 vport_ops_list, 如果那個ops的 type跟傳進來的type是一樣的，那就呼叫對應的 create function.
1
2
3
4
5
6
7
8
9
10
static const struct vport_ops *vport_ops_list[] = {
        &ovs_netdev_vport_ops,
        &ovs_internal_vport_ops,
#if IS_ENABLED(CONFIG_NET_IPGRE_DEMUX)
        &ovs_gre_vport_ops,
        &ovs_gre64_vport_ops,
#endif
        &ovs_vxlan_vport_ops,
        &ovs_lisp_vport_ops,
};
  • vport_ops 有多種type.這邊我們專注於 ovs_netdev_vport_ops這種type.
1
2
3
4
5
6
7
const struct vport_ops ovs_netdev_vport_ops = {
        .type           = OVS_VPORT_TYPE_NETDEV,
        .create         = netdev_create,
        .destroy        = netdev_destroy,
        .get_name       = ovs_netdev_get_name,
        .send           = netdev_send,
};
  • vport當type是 netdev的時候，成員的function pointer就會按此設定為各個function.
  • 可以看到他的 create 對應到 netdev_create
1
2
3
4
5
static struct vport *netdev_create(const struct vport_parms *parms)
...
        err = netdev_rx_handler_register(netdev_vport->dev, netdev_frame_hook,
                                         vport);
...
  • 這邊註冊netdevice的receive handler, handler為 netdev_frame_hook
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
int netdev_rx_handler_register(struct net_device *dev,
                                rx_handler_func_t *rx_handler,
                                void *rx_handler_data)
{
         ASSERT_RTNL();

         if (dev->rx_handler)
                 return -EBUSY;

         /* Note: rx_handler_data must be set before rx_handler */
         rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
         rcu_assign_pointer(dev->rx_handler, rx_handler);

        return 0;
}
  • 這邊會把設定該 net_device中的兩個pointer
  • rx_handler_data 這邊就設定成該vport.
  • rx_handler 接收到封包的處理函式，這邊就是netdev_frame_hook。

個人資訊

我目前於 Hiskio 平台上面有開設 Kubernetes 相關課程，歡迎有興趣的人參考並分享，裡面有我從底層到實戰中對於 Kubernetes 的各種想法

組合包
https://hiskio.com/packages/mLwKb3zrx

疑難雜症除錯篇
https://hiskio.com/courses/440/about?promo_code=VEQ4XNG

單堂(CI/CD)
https://hiskio.com/courses/385?promo_code=13K49YE&p=blog1

基礎概念
https://hiskio.com/courses/349?promo_code=13LY5RE

另外，歡迎按讚加入我個人的粉絲專頁，裡面會定期分享各式各樣的文章，有的是翻譯文章，也有部分是原創文章，主要會聚焦於 CNCF 領域
https://www.facebook.com/technologynoteniu

如果有使用 Telegram 的也可以訂閱下列頻道來，裡面我會定期推播通知各類文章
https://t.me/technologynote

你的捐款將給予我文章成長的動力

Related Posts

Welcome to my other publishing channels