Unable to access local service(opened :::9100) via local IP

by Mason Kim   Last Updated September 11, 2019 17:01 PM

There is something really weird.
While I can access local service opened :::9100 via localhost,

[root@os3 ~]# curl localhost:9100
<html>
<head><title>Node Exporter</title></head>
<body>
<h1>Node Exporter</h1>
<p><a href="/metrics">Metrics</a></p>
</body>
</html>
[root@os3 ~]#

I can't access all services via local IP (except for SSH)

[root@os3 ~]# curl 70.60.31.103:9100
.... hanging ....

Server info
OS: Centos 7.6
Opened services

[root@os3 ~]# netstat -tpln
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:111             0.0.0.0:*               LISTEN      1/systemd
tcp        0      0 0.0.0.0:6000            0.0.0.0:*               LISTEN      12116/X
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      12890/sshd
tcp        0      0 127.0.0.1:631           0.0.0.0:*               LISTEN      12886/cupsd
tcp        0      0 127.0.0.1:3128          0.0.0.0:*               LISTEN      15722/sshd: hbseo@p
tcp        0      0 127.0.0.1:25            0.0.0.0:*               LISTEN      13262/master
tcp6       0      0 :::9100                 :::*                    LISTEN      14128/node_exporter
tcp6       0      0 :::111                  :::*                    LISTEN      1/systemd
tcp6       0      0 :::6000                 :::*                    LISTEN      12116/X
tcp6       0      0 :::22                   :::*                    LISTEN      12890/sshd
tcp6       0      0 ::1:631                 :::*                    LISTEN      12886/cupsd
tcp6       0      0 ::1:3128                :::*                    LISTEN      15722/sshd: hbseo@p
tcp6       0      0 ::1:25                  :::*                    LISTEN      13262/master

[root@os3 ~]#

Interfaces

[root@os3 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: enp5s0f0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 0c:c4:7a:69:21:b2 brd ff:ff:ff:ff:ff:ff
3: enp5s0f1: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 0c:c4:7a:69:21:b3 brd ff:ff:ff:ff:ff:ff
4: enp130s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 0c:c4:7a:55:fe:a2 brd ff:ff:ff:ff:ff:ff
    inet 70.60.31.103/24 brd 70.60.31.255 scope global noprefixroute enp130s0f0
       valid_lft forever preferred_lft forever
    inet6 fe80::bc7c:de99:848a:a6ff/64 scope link noprefixroute
       valid_lft forever preferred_lft forever
5: enp4s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 0c:c4:7a:bc:71:68 brd ff:ff:ff:ff:ff:ff
6: enp130s0f1: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 0c:c4:7a:55:fe:a3 brd ff:ff:ff:ff:ff:ff
7: enp4s0f1: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 0c:c4:7a:bc:71:69 brd ff:ff:ff:ff:ff:ff
8: enp133s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 0c:c4:7a:bc:71:6e brd ff:ff:ff:ff:ff:ff
9: enp133s0f1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 0c:c4:7a:bc:71:6f brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.103/24 brd 192.168.1.255 scope global noprefixroute enp133s0f1
       valid_lft forever preferred_lft forever
    inet6 fe80::2fdc:d6c1:e4f3:2c8/64 scope link noprefixroute
       valid_lft forever preferred_lft forever
10: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
    link/ether 02:42:17:99:89:9c brd ff:ff:ff:ff:ff:ff
    inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
       valid_lft forever preferred_lft forever
11: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 9e:e6:48:43:73:87 brd ff:ff:ff:ff:ff:ff
12: br-tun: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether ea:c1:6d:7d:8d:41 brd ff:ff:ff:ff:ff:ff
13: br-int: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether ce:14:02:e1:0d:4b brd ff:ff:ff:ff:ff:ff

Routing tables

[root@os3 ~]# route
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
default         gateway         0.0.0.0         UG    100    0        0 enp130s0f0
70.60.31.0      0.0.0.0         255.255.255.0   U     100    0        0 enp130s0f0
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
192.168.1.0     0.0.0.0         255.255.255.0   U     101    0        0 enp133s0f1

[root@os3 ~]# ip route show table local
broadcast 70.60.31.0 dev enp130s0f0 proto kernel scope link src 70.60.31.103
local 70.60.31.103 dev enp130s0f0 proto kernel scope host src 70.60.31.103
broadcast 70.60.31.255 dev enp130s0f0 proto kernel scope link src 70.60.31.103
broadcast 127.0.0.0 dev lo proto kernel scope link src 127.0.0.1
local 127.0.0.0/8 dev lo proto kernel scope host src 127.0.0.1
local 127.0.0.1 dev lo proto kernel scope host src 127.0.0.1
broadcast 127.255.255.255 dev lo proto kernel scope link src 127.0.0.1
broadcast 172.17.0.0 dev docker0 proto kernel scope link src 172.17.0.1
local 172.17.0.1 dev docker0 proto kernel scope host src 172.17.0.1
broadcast 172.17.255.255 dev docker0 proto kernel scope link src 172.17.0.1
broadcast 192.168.1.0 dev enp133s0f1 proto kernel scope link src 192.168.1.103
local 192.168.1.103 dev enp133s0f1 proto kernel scope host src 192.168.1.103
broadcast 192.168.1.255 dev enp133s0f1 proto kernel scope link src 192.168.1.103
[root@os3 ~]#
[root@os3 ~]#
[root@os3 ~]# ip route show table main
default via 70.60.31.1 dev enp130s0f0 proto static metric 100
70.60.31.0/24 dev enp130s0f0 proto kernel scope link src 70.60.31.103 metric 100
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1
192.168.1.0/24 dev enp133s0f1 proto kernel scope link src 192.168.1.103 metric 101
[root@os3 ~]#

bridge info

[root@os3 ~]# brctl show
bridge name     bridge id               STP enabled     interfaces
docker0         8000.02421799899c       no
[root@os3 ~]#

iptables

[root@os3 ~]# iptables -t nat -vL
Chain PREROUTING (policy ACCEPT 482 packets, 53615 bytes)
 pkts bytes target     prot opt in     out     source               destination
   95  5700 DOCKER     all  --  any    any     anywhere             anywhere             ADDRTYPE match dst-type LOCAL

Chain INPUT (policy ACCEPT 313 packets, 29974 bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain OUTPUT (policy ACCEPT 805 packets, 53019 bytes)
 pkts bytes target     prot opt in     out     source               destination
    7   420 DOCKER     all  --  any    any     anywhere            !loopback/8           ADDRTYPE match dst-type LOCAL

Chain POSTROUTING (policy ACCEPT 805 packets, 53019 bytes)
 pkts bytes target     prot opt in     out     source               destination
    0     0 MASQUERADE  all  --  any    !docker0  172.17.0.0/16        anywhere

Chain DOCKER (2 references)
 pkts bytes target     prot opt in     out     source               destination
    0     0 RETURN     all  --  docker0 any     anywhere             anywhere
[root@os3 ~]# iptables -vL
Chain INPUT (policy ACCEPT 47446 packets, 26M bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain FORWARD (policy DROP 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination
    0     0 DOCKER-USER  all  --  any    any     anywhere             anywhere
    0     0 DOCKER-ISOLATION-STAGE-1  all  --  any    any     anywhere             anywhere
    0     0 ACCEPT     all  --  any    docker0  anywhere             anywhere             ctstate RELATED,ESTABLISHED
    0     0 DOCKER     all  --  any    docker0  anywhere             anywhere
    0     0 ACCEPT     all  --  docker0 !docker0  anywhere             anywhere
    0     0 ACCEPT     all  --  docker0 docker0  anywhere             anywhere

Chain OUTPUT (policy ACCEPT 39943 packets, 41M bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain DOCKER (1 references)
 pkts bytes target     prot opt in     out     source               destination

Chain DOCKER-ISOLATION-STAGE-1 (1 references)
 pkts bytes target     prot opt in     out     source               destination
    0     0 DOCKER-ISOLATION-STAGE-2  all  --  docker0 !docker0  anywhere             anywhere
    0     0 RETURN     all  --  any    any     anywhere             anywhere

Chain DOCKER-ISOLATION-STAGE-2 (1 references)
 pkts bytes target     prot opt in     out     source               destination
    0     0 DROP       all  --  any    docker0  anywhere             anywhere
    0     0 RETURN     all  --  any    any     anywhere             anywhere

Chain DOCKER-USER (1 references)
 pkts bytes target     prot opt in     out     source               destination
    0     0 RETURN     all  --  any    any     anywhere             anywhere
[root@os3 ~]#

Is there something I am missing?



Related Questions


Updated October 26, 2015 07:00 AM

Updated April 23, 2016 08:00 AM

Updated May 29, 2015 23:00 PM

Updated June 01, 2015 03:00 AM

Updated March 30, 2018 04:01 AM