WAN failover - routes flapping

NetworqAndy · Fri Apr 12, 2024 3:00 pm

Hi all,

I'm having issues creating a set of mangle rules for a router which will have 2x separate WAN connections (eventually 3) for redundancy. Assume all three WAN connections will be dynamic IPs at this point.

It's only partly setup at the moment but the basic background for this setup:
3 VLANs - 20 will be guest wireless, 30 IP phones, and 40 will be CCTV.
Ether1 and Ether2 are WAN1 and WAN2 respectively.

Up until I created the routing tables there was something odd going on where I had internet access from a client machine via WAN1, but if I disabled WAN1 (forcing all traffic via WAN2), it would only work for another 15-20 seconds before WAN2 was unusable. I've made a change somehow which has resulted in a situation where there is flapping having recreated the IP routes. If I access config via IP address (rather than MAC address) it will connect for 10 seconds then disconnect for 20, then back again.

It's only partially setup still, so there's plenty to do, but just trying to set things up logically one step at a time!

Grateful of any pointers

Andy

# apr/12/2024 12:45:28 by RouterOS 6.49.13
# software id = E75X-80RJ
#
# model = RB1100Dx4
# serial number = HEY09AX5MMT
/interface bridge
add name=bridge1
/interface ethernet
set [ find default-name=ether1 ] name=WAN1
set [ find default-name=ether2 ] name=WAN2
/interface vlan
add interface=bridge1 name=vlan20_Guest vlan-id=20
add interface=bridge1 name=vlan30_VOIP vlan-id=30
add interface=bridge1 name=vlan40_CCTV vlan-id=40
add interface=bridge1 name=vlan90_WAN3 vlan-id=90
/caps-man datapath
add bridge=bridge1 name=datapath_StaffMGMT
add bridge=bridge1 name=datapath_Guest vlan-id=20 vlan-mode=use-tag
/caps-man security
add authentication-types=wpa2-psk encryption=aes-ccm name=security_StaffMGMT
add authentication-types=wpa2-psk encryption=aes-ccm name=security_Guest
/caps-man configuration
add country="united kingdom" datapath=datapath_Guest datapath.client-to-client-forwarding=no datapath.vlan-id=20 datapath.vlan-mode=use-tag installation=indoor mode=ap name=cfg_GuestWifi security=security_Guest ssid=OldMill_GuestWiFi
add country="united kingdom" datapath=datapath_StaffMGMT datapath.bridge=bridge1 installation=indoor mode=ap name=cfg_StaffMGMT security=security_StaffMGMT ssid=OldMill_Staff
/interface ethernet switch port
set 0 default-vlan-id=0
set 1 default-vlan-id=0
set 2 default-vlan-id=0
set 3 default-vlan-id=0
set 4 default-vlan-id=0
set 5 default-vlan-id=0
set 6 default-vlan-id=0
set 7 default-vlan-id=0
set 8 default-vlan-id=0
set 9 default-vlan-id=0
set 10 default-vlan-id=0
set 11 default-vlan-id=0
set 12 default-vlan-id=0
set 13 default-vlan-id=0
set 14 default-vlan-id=0
set 15 default-vlan-id=0
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip dhcp-server
add disabled=no interface=vlan30_VOIP lease-time=4w2d name=dhcpVOIP
add disabled=no interface=vlan40_CCTV lease-time=4w2d name=dhcpCCTV
/ip pool
add name=dhcp_StaffMGMT ranges=10.10.100.1-10.10.199.254
add name=dhcp_Guest ranges=10.20.100.1-10.20.199.254
add name=dhcp_VOIP ranges=10.30.100.1-10.30.199.254
add name=dhcp_CCTV ranges=10.40.100.1-10.40.199.254
/ip dhcp-server
add address-pool=dhcp_StaffMGMT disabled=no interface=bridge1 lease-time=1w3d name=dhcpStaffMGMT
add address-pool=dhcp_Guest disabled=no interface=vlan20_Guest lease-time=1d name=dhcpGuest
/caps-man manager
set enabled=yes package-path=/ upgrade-policy=suggest-same-version
/caps-man manager interface
set [ find default=yes ] forbid=yes
add disabled=no interface=bridge1
add disabled=no
/caps-man provisioning
add action=create-dynamic-enabled master-configuration=cfg_StaffMGMT name-format=identity slave-configurations=cfg_GuestWifi
/dude
set enabled=yes
/interface bridge port
add bridge=bridge1 interface=ether3
add bridge=bridge1 interface=ether4
add bridge=bridge1 interface=ether5
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether6
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether7
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether8
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether9
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether10
add bridge=bridge1 interface=ether11
add bridge=bridge1 interface=ether12
add bridge=bridge1 interface=ether13
add bridge=bridge1 interface=vlan20_Guest
add bridge=bridge1 interface=vlan30_VOIP
add bridge=bridge1 interface=vlan40_CCTV
/ip address
add address=10.30.0.1/16 interface=vlan30_VOIP network=10.30.0.0
add address=10.40.0.1/16 interface=vlan40_CCTV network=10.40.0.0
add address=10.10.0.1/16 interface=bridge1 network=10.10.0.0
add address=10.20.0.1/16 interface=vlan20_Guest network=10.20.0.0
/ip dhcp-client
add add-default-route=no disabled=no interface=WAN1
add add-default-route=no disabled=no interface=WAN2
/ip dhcp-server lease
add address=10.10.0.11 mac-address=78:9A:18:2D:87:60 server=dhcpStaffMGMT
add address=10.10.0.101 client-id=1:d4:1:c3:57:47:2e mac-address=D4:01:C3:57:47:2E server=dhcpStaffMGMT
add address=10.10.0.102 client-id=1:d4:1:c3:57:20:3f mac-address=D4:01:C3:57:20:3F server=dhcpStaffMGMT
/ip dhcp-server network
add address=10.10.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.10.0.1
add address=10.20.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.20.0.1
add address=10.30.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.30.0.1
add address=10.40.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.40.0.1
/ip dns
set servers=8.8.8.8,8.8.4.4
/ip firewall address-list
add address=10.20.100.1-10.20.199.254 list=Guest
/ip firewall filter
add action=drop chain=input comment="Block guest access to router local ports" dst-address=10.20.0.0/16 dst-port=80,21,22,23,8291 protocol=tcp src-address-list=Guest
add action=drop chain=input dst-address=10.10.0.0/16 src-address-list=Guest
add action=drop chain=input dst-address=10.30.0.0/16 src-address-list=Guest
add action=drop chain=input dst-address=10.40.0.0/16 src-address-list=Guest
/ip firewall mangle
add action=mark-connection chain=prerouting connection-mark=no-mark in-interface=WAN1 new-connection-mark=viaWAN1 passthrough=yes
add action=mark-connection chain=prerouting connection-mark=no-mark in-interface=WAN2 new-connection-mark=viaWAN2 passthrough=yes
add action=mark-routing chain=prerouting connection-mark=viaWAN1 new-routing-mark=viaWAN1 passthrough=no
add action=mark-routing chain=prerouting connection-mark=viaWAN2 new-routing-mark=viaWAN2 passthrough=no
add action=mark-connection chain=prerouting connection-mark=no-mark dst-address-type=!local in-interface=bridge1 new-connection-mark=viaWAN1 passthrough=yes per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=prerouting connection-mark=no-mark dst-address-type=!local in-interface=bridge1 new-connection-mark=viaWAN2 passthrough=yes per-connection-classifier=both-addresses:1/0
add action=fasttrack-connection chain=forward connection-mark=no-mark connection-state=established,related
/ip firewall nat
add action=masquerade chain=srcnat out-interface=WAN1 src-address=10.10.0.0/16
add action=masquerade chain=srcnat out-interface=WAN2 src-address=10.10.0.0/16
add action=masquerade chain=srcnat out-interface=WAN1 src-address=10.20.0.0/16
add action=masquerade chain=srcnat out-interface=WAN2 src-address=10.20.0.0/16
add action=masquerade chain=srcnat out-interface=WAN1 src-address=10.30.0.0/16
add action=masquerade chain=srcnat out-interface=WAN2 src-address=10.30.0.0/16
add action=masquerade chain=srcnat out-interface=WAN1 src-address=10.40.0.0/16
add action=masquerade chain=srcnat out-interface=WAN2 src-address=10.40.0.0/16
/ip route
add distance=1 gateway=WAN1 routing-mark=viaWAN1
add check-gateway=ping distance=5 gateway=WAN1 routing-mark=viaWAN1
add distance=1 gateway=WAN2 routing-mark=viaWAN2
add distance=10 gateway=WAN2 routing-mark=viaWAN2
/ip route rule
add table=viaWAN1
add table=viaWAN2
add table=viaWAN3
/system clock
set time-zone-name=Europe/London
/system identity
set name=RB1100-Reception

TheCat12 · Fri Apr 12, 2024 5:39 pm

There are some missing and incorrectly configured rules. I'll post them edited and in the correct order after which I will explain the changes:

/ip firewall mangle
add action=mark-connection chain=prerouting connection-mark=no-mark in-interface=WAN1 new-connection-mark=viaWAN1 passthrough=yes
add action=mark-connection chain=prerouting connection-mark=no-mark in-interface=WAN2 new-connection-mark=viaWAN2 passthrough=yes
add action=mark-routing chain=prerouting connection-mark=viaWAN1 in-interface=bridge1 new-routing-mark=viaWAN1 passthrough=yes
add action=mark-routing chain=output connection-mark=viaWAN1 new-routing-mark=viaWAN1 passthrough=no
add action=mark-routing chain=prerouting connection-mark=viaWAN2 in-interface=bridge1 new-routing-mark=viaWAN2 passthrough=yes
add action=mark-routing chain=output connection-mark=viaWAN2 new-routing-mark=viaWAN2 passthrough=no
add action=mark-connection chain=prerouting connection-mark=no-mark dst-address-type=!local in-interface=bridge1 new-connection-mark=viaWAN1 passthrough=yes per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=prerouting connection-mark=no-mark dst-address-type=!local in-interface=bridge1 new-connection-mark=viaWAN2 passthrough=yes per-connection-classifier=both-addresses:2/1

1. There were missing output rules
2. The PCC on the second rule was wrong - 1/0 instead of 2/1

On the NAT side you could use just two NAT rules:

/ip firewall nat
add action=masquerade chain=srcnat out-interface=WAN1
add action=masquerade chain=srcnat out-interface=WAN2

There's also something off with the routes. The ones with high distances aren't needed. Everything should look like that:

/ip route
add dst-address=0.0.0.0/0 gateway=WAN1 routing-mark=viaWAN1 check-gateway=ping
add dst-address=0.0.0.0/0 gateway=WAN2 routing-mark=viaWAN2 check-gateway=ping
add distance=1 dst-address=0.0.0.0/0 gateway="WAN1_gateway_address" check-gateway=ping
add distance=2 dst-address=0.0.0.0/0 gateway="WAN2_gateway_address" check-gateway=ping

NetworqAndy · Fri Apr 12, 2024 7:52 pm

Hi @TheCat12 - thanks for your reply. Feel a bit daft missing the output mangle rules. Also a good call on simplifying the NAT rules. Sadly those changes haven't really made much difference.

...which has made me wonder if there's something deeper going on here. I still can't connect via IP address to the router. In fact I can't even ping the router (10.0.0.1). From within winbox I can get the router to ping 8.8.8.8 via either WAN1 or WAN2, however from the bridge it's still failing to ping.

Not sure if I've messed up something basic somewhere else. I've double-checked the firewall rules and can't see an issue (plus have temporarily disabled them). I'm connected via Ether11 which is a member of bridge1 which is set to admit all on VLAN1.

Grateful of any further insights as to how this has gone wrong so quickly

Andy

TheCat12 · Fri Apr 12, 2024 8:43 pm

Now that you mention it, there is also something wrong with the VLAN configuration. Could you make a network diagram with the VLANs included or at least tell me which ports are access ones (if there are such) and which are trunk?

NetworqAndy · Fri Apr 12, 2024 9:16 pm

Hi,

From this particular main router's perspective, ports 5-10 will end up being the VLAN trunks.
I've left ports 3-5 for possible additional WAN usage down the line, and ports 11-13 for local access to VLAN 1 (10.10.0.0 network).
VLANs over the trunk ports will be 1, 20, 30, 40 and 90.
No ports on this router will be set to VLAN access ports - there's switches further down the line that will deal with all this and hAP routers to provide staff and guest internet (VLAN'd accordingly).

I haven't concentrated on finishing this bit of the config yet - with a single WAN connection and simple NAT rule the routing/DHCP/internet was working on each of the VLANs on access ports I've created on one of the switches on the benchtop setup on my desk. It's only really gone to pot since I created the mangle rules, but that's not to say I've not mistakenly deleted part of the config well away from that with sausage fingers

Andy

TheCat12 · Fri Apr 12, 2024 10:06 pm

Hi, thank you for informing me about your VLAN setup. Now I can give you some suggestions regarding it and the mangle situation. Hopefully it won't be a big fuss if I introduce a new VLAN in place of VLAN1 for easier management.

/interface vlan add interface=bridge1 name=vlan10_StaffMGMT vlan-id=10

/ip address set [ find interface=bridge1 ] interface=vlan10_StaffMGMT

/ip dhcp-server set [ find name=dhcpStaffMGMT ] interface=vlan10_StaffMGMT

/interface bridge port
set [ find interface=ether5 ] frame-types=admit-only-vlan-tagged
set [ find interface=ether11 ] pvid=10
set [ find interface=ether12 ] pvid=10
set [ find interface=ether13 ] pvid=10
remove [ find interface=vlan20_Guest ]
remove [ find interface=vlan30_VOIP ]
remove [ find interface=vlan40_CCTV ]

/interface bridge vlan
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9,ether10 vlan-ids=10
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9,ether10 vlan-ids=20
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9,ether10 vlan-ids=30
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9,ether10 vlan-ids=40
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9,ether10 vlan-ids=90

/interface bridge set bridge1 vlan-filtering=yes

/interface list add name=VLAN

/interface list member
add list=VLAN interface=vlan10_StaffMGMT
add list=VLAN interface=vlan20_Guest
add list=VLAN interface=vlan30_VOIP
add list=VLAN interface=vlan40_CCTV

/ip firewall mangle set [ find in-interface=bridge1 ] in-interface-list=VLAN

Though these settings the VLAN will be configured properly and all networks will be isolated from one another and hopefully the mangle rules will start to work as intended

NetworqAndy · Sat Apr 13, 2024 12:22 am

Hi,

Thanks again for that - creating the trunks and a management VLAN was on the to do list.

Alas... still no luck. There's something fundamentally wrong here with the bridge. If I connect from a computer on the WAN side of the network, there's no issues pinging or Winboxing in by IP address. Put a machine on the LAN side of the bridge and while an IP address is dished out over DHCP, but no way to ping/winbox in by IP, only by MAC address.

Also I'm still unable to ping Google's DNS servers from within the router when forcing the ping to the bridge1 interface (WAN1/WAN2 work fine).

I've hit a brick wall for today and switched off and walked away. Massively grateful of the support so far @TheCat12 but sadly still something fundamentally annoyingly wrong! I know it'll be simple, but I just can't see the issue.... yet!!

FYI Current config now looks like:

# apr/12/2024 22:52:01 by RouterOS 6.49.13
# software id = E75X-80RJ
#
# model = RB1100Dx4
# serial number = HEY09AX5MMT
/interface bridge
add name=bridge1 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether1 ] name=WAN1
set [ find default-name=ether2 ] name=WAN2
/interface vlan
add interface=bridge1 name=vlan10_StaffMGMT vlan-id=10
add interface=bridge1 name=vlan20_Guest vlan-id=20
add interface=bridge1 name=vlan30_VOIP vlan-id=30
add interface=bridge1 name=vlan40_CCTV vlan-id=40
add interface=bridge1 name=vlan90_WAN3 vlan-id=90
/caps-man datapath
add bridge=bridge1 name=datapath_StaffMGMT
add bridge=bridge1 name=datapath_Guest vlan-id=20 vlan-mode=use-tag
/caps-man security
add authentication-types=wpa2-psk encryption=aes-ccm name=security_StaffMGMT
add authentication-types=wpa2-psk encryption=aes-ccm name=security_Guest
/caps-man configuration
add country="united kingdom" datapath=datapath_Guest \
    datapath.client-to-client-forwarding=no datapath.vlan-id=20 \
    datapath.vlan-mode=use-tag installation=indoor mode=ap name=cfg_GuestWifi \
    security=security_Guest ssid=OldMill_GuestWiFi
add country="united kingdom" datapath=datapath_StaffMGMT datapath.bridge=\
    bridge1 installation=indoor mode=ap name=cfg_StaffMGMT security=\
    security_StaffMGMT ssid=OldMill_Staff
/interface ethernet switch port
set 0 default-vlan-id=0
set 1 default-vlan-id=0
set 2 default-vlan-id=0
set 3 default-vlan-id=0
set 4 default-vlan-id=0
set 5 default-vlan-id=0
set 6 default-vlan-id=0
set 7 default-vlan-id=0
set 8 default-vlan-id=0
set 9 default-vlan-id=0
set 10 default-vlan-id=0
set 11 default-vlan-id=0
set 12 default-vlan-id=0
set 13 default-vlan-id=0
set 14 default-vlan-id=0
set 15 default-vlan-id=0
/interface list
add name=VLAN
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip dhcp-server
add disabled=no interface=vlan30_VOIP lease-time=4w2d name=dhcpVOIP
add disabled=no interface=vlan40_CCTV lease-time=4w2d name=dhcpCCTV
/ip pool
add name=dhcp_StaffMGMT ranges=10.10.100.1-10.10.199.254
add name=dhcp_Guest ranges=10.20.100.1-10.20.199.254
add name=dhcp_VOIP ranges=10.30.100.1-10.30.199.254
add name=dhcp_CCTV ranges=10.40.100.1-10.40.199.254
/ip dhcp-server
add address-pool=dhcp_StaffMGMT disabled=no interface=vlan10_StaffMGMT \
    lease-time=1w3d name=dhcpStaffMGMT
add address-pool=dhcp_Guest disabled=no interface=vlan20_Guest lease-time=1d \
    name=dhcpGuest
/caps-man manager
set enabled=yes package-path=/ upgrade-policy=suggest-same-version
/caps-man manager interface
set [ find default=yes ] forbid=yes
add disabled=no interface=bridge1
add disabled=no
/caps-man provisioning
add action=create-dynamic-enabled master-configuration=cfg_StaffMGMT \
    name-format=identity slave-configurations=cfg_GuestWifi
/dude
set enabled=yes
/interface bridge port
add bridge=bridge1 interface=ether3
add bridge=bridge1 interface=ether4
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether5
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether6
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether7
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether8
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether9
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether10
add bridge=bridge1 interface=ether11 pvid=10
add bridge=bridge1 interface=ether12 pvid=10
add bridge=bridge1 interface=ether13 pvid=10
/interface bridge vlan
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9 \
    untagged=ether10 vlan-ids=10
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9 \
    untagged=ether10 vlan-ids=20
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9 \
    untagged=ether10 vlan-ids=30
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9 \
    untagged=ether10 vlan-ids=40
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9 \
    untagged=ether10 vlan-ids=90
/interface list member
add interface=vlan10_StaffMGMT list=VLAN
add interface=vlan20_Guest list=VLAN
add interface=vlan30_VOIP list=VLAN
add interface=vlan40_CCTV list=VLAN
/ip address
add address=10.30.0.1/16 interface=vlan30_VOIP network=10.30.0.0
add address=10.40.0.1/16 interface=vlan40_CCTV network=10.40.0.0
add address=10.10.0.1/16 interface=vlan10_StaffMGMT network=10.10.0.0
add address=10.20.0.1/16 interface=vlan20_Guest network=10.20.0.0
/ip dhcp-client
add add-default-route=no disabled=no interface=WAN1
add add-default-route=no disabled=no interface=WAN2
/ip dhcp-server lease
add address=10.10.0.11 mac-address=78:9A:18:2D:87:60 server=dhcpStaffMGMT
add address=10.10.0.101 client-id=1:d4:1:c3:57:47:2e mac-address=\
    D4:01:C3:57:47:2E server=dhcpStaffMGMT
add address=10.10.0.102 client-id=1:d4:1:c3:57:20:3f mac-address=\
    D4:01:C3:57:20:3F server=dhcpStaffMGMT
/ip dhcp-server network
add address=10.10.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.10.0.1
add address=10.20.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.20.0.1
add address=10.30.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.30.0.1
add address=10.40.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.40.0.1
/ip dns
set servers=8.8.8.8,8.8.4.4
/ip firewall address-list
add address=10.20.100.1-10.20.199.254 list=Guest
/ip firewall filter
add action=drop chain=input comment=\
    "Block guest access to router local ports" disabled=yes dst-address=\
    10.20.0.0/16 dst-port=80,21,22,23,8291 protocol=tcp src-address-list=\
    Guest
add action=drop chain=input disabled=yes dst-address=10.10.0.0/16 \
    src-address-list=Guest
add action=drop chain=input disabled=yes dst-address=10.30.0.0/16 \
    src-address-list=Guest
add action=drop chain=input disabled=yes dst-address=10.40.0.0/16 \
    src-address-list=Guest
add action=fasttrack-connection chain=forward connection-state=\
    established,related
add action=accept chain=input connection-state=established,related
add action=accept chain=forward connection-state=established,related
add action=accept chain=input protocol=icmp
/ip firewall mangle
add action=mark-connection chain=prerouting connection-mark=no-mark \
    in-interface=WAN1 new-connection-mark=viaWAN1 passthrough=yes
add action=mark-connection chain=prerouting connection-mark=no-mark \
    in-interface=WAN2 new-connection-mark=viaWAN2 passthrough=yes
add action=mark-routing chain=prerouting connection-mark=viaWAN1 \
    new-routing-mark=viaWAN1 passthrough=yes
add action=mark-routing chain=output connection-mark=viaWAN1 \
    new-routing-mark=viaWAN1 passthrough=no
add action=mark-routing chain=prerouting connection-mark=viaWAN2 \
    new-routing-mark=viaWAN2 passthrough=yes
add action=mark-routing chain=output connection-mark=viaWAN2 \
    new-routing-mark=viaWAN2 passthrough=no
add action=mark-connection chain=prerouting connection-mark=no-mark \
    dst-address-type=!local in-interface=bridge1 new-connection-mark=viaWAN1 \
    passthrough=yes per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=prerouting connection-mark=no-mark \
    dst-address-type=!local in-interface=bridge1 new-connection-mark=viaWAN2 \
    passthrough=yes per-connection-classifier=both-addresses:2/1
add action=fasttrack-connection chain=forward connection-mark=no-mark \
    connection-state=established,related disabled=yes
/ip firewall nat
add action=masquerade chain=srcnat out-interface=WAN1
add action=masquerade chain=srcnat out-interface=WAN2
add action=masquerade chain=srcnat disabled=yes out-interface=WAN1 \
    src-address=10.10.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN2 \
    src-address=10.10.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN1 \
    src-address=10.20.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN2 \
    src-address=10.20.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN1 \
    src-address=10.30.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN2 \
    src-address=10.30.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN1 \
    src-address=10.40.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN2 \
    src-address=10.40.0.0/16
/ip route
add check-gateway=ping distance=1 gateway=WAN1 routing-mark=viaWAN1
add check-gateway=ping distance=1 gateway=WAN2 routing-mark=viaWAN2
add check-gateway=ping distance=1 dst-address=0.0.0.0/24 gateway=192.168.2.1
/ip route rule
add table=viaWAN1
add table=viaWAN2
add table=viaWAN3
/system clock
set time-zone-name=Europe/London
/system identity
set name=RB1100-Reception

TheCat12 · Sat Apr 13, 2024 8:55 am

Hi, there are some tweaks that I made in the config but don't see on the current. They were the following:

1. Using in-interface-list=VLAN instead of in-interface=bridge1

2. Adding another route in the main table for WAN2 with distance=2

/ip firewall mangle
set [ find in-interface=bridge1] in-interface="" in-interface-list=VLAN

/ip route
add distance=2 dst-address=0.0.0.0/0 gateway="WAN2_gateway_address" routing-table=main

TheCat12 · Sat Apr 13, 2024 9:38 am

Also I found a possible solution here:

viewtopic.php?f=2&t=136969&p=674653#p674653

So overall following changes should be made:

/interface list
set [ find name=VLAN ] name=all_LANs

/interface list member add list=all_LANs interface=bridge1

/ip firewall address-list
add list=local address=10.10.100.1/24
add list=local address=10.20.100.1/24
add list=local address=10.30.100.1/24
add list=local address=10.40.100.1/24

/ip firewall mangle
set [ find in-interface=bridge1 ] dst-address-list=!local dst-address-type="" in-interface="" in-interface-list=all_LANs

/ip route add distance=2 dst-address=0.0.0.0/0 gateway="WAN2_gateway_address" check-gateway=ping

anav · Sat Apr 13, 2024 7:54 pm

The fundamentals are the same for most setups, the extra sauce stems from well communicated traffic requirements.
You have two WANS, in the future 3 possibley four wans.
You havent specified a clear plan. WHich is primary and which is secondary so assuming WAN1 is primary and WAN2 is backup.
If WAN1 goes down you want all users to have access to WAN2.
THe additional aspect is using the wired router also as capsman, which I know nothing about.
Assuming it has no bearing on other RB1100 config elements...............

I see you have Four VLANS, thus expect, Four IP addresses, Four Pools, Four DHCP SErvers, Four DCHP Server-Networks.
What is unclear is what is the purpose of VLAN90..
If this is in anticipation of a WAN connection where the ISP brings the internet connection on vlan90, fine, thus change the above to THREE of everything.
+++++++++++++++++++++++++++++
Capstuff ignored.
++++++++++++++++++++++++++++
Only three DHCP SERvers noted, where is DCHP server for vlan 20 ??
Four Pools?? Now I am confused I see guest/voip/cctv/ and STAFF MGMT??

There seems to be a disconnect you have vlan90 supposedly for WAN3 yet no vlan for STAFF MGTM,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
AHH I SEE, you are mixing APPLES and ORANGES>
Once you start using vlans, do not use the bridge for any assignments, just bridging.............. vlan10 will be STAFF MGMT.
Why do you add vlans to /interface bridge ports?? Normally its for etherports and wlan ports....
In your latest iteration you completely forgot about /interface bridge vlans --? missing!

OKAY the below solution is slightly modified for future WANS on port 3 and port4, port5 will bew off bridge access, in case bridge burps on you and actually a safe spot to to any config.
Assuming ports 11,12,13 are staff ports.

Only need four mangle rules as you have not incoming to the router services itself (VPN), nor are you port forwarding to servers on any wan.

You need manual routes through the main table as well as pcc routes. ( routing rules not required )

.....................

# model = RB1100Dx4
# serial number = HEY09AX5MMT
/interface bridge
add name=bridge1
/interface ethernet
set [ find default-name=ether1 ] name=WAN1
set [ find default-name=ether2 ] name=WAN2
set [ find default-name=ether3 ] name=WAN3-Future
set [ find default-name=ether4 ] name=WAN4-Future
set [ find default-name=ether5 ] name=Off-Bridge5
/interface vlan
add interface=bridge1 name=vlan10_Staff vlan-id=10
add interface=bridge1 name=vlan20_Guest vlan-id=20
add interface=bridge1 name=vlan30_VOIP vlan-id=30
add interface=bridge1 name=vlan40_CCTV vlan-id=40
add interface=bridge1 name=vlan90_WAN3 vlan-id=90 comment=FutureWAN
/interface list
add name=WAN
add name=LAN
add name=MGMT
/routing table
add fib name=useWAN1
add fib name=useWAN2
/ip dhcp-server
add disabled=no interface=vlan10_Staff lease-time=4w2d name=dhcpStaff
add disabled=no interface=vlan20-Guest lease-time=4w2d name=dhcpGuest
add disabled=no interface=vlan30_VOIP lease-time=4w2d name=dhcpVOIP
add disabled=no interface=vlan40_CCTV lease-time=4w2d name=dhcpCCTV
/ip pool
add name=dhcp_StaffMGMT ranges=10.10.100.1-10.10.199.254
add name=dhcp_Guest ranges=10.20.100.1-10.20.199.254
add name=dhcp_VOIP ranges=10.30.100.1-10.30.199.254
add name=dhcp_CCTV ranges=10.40.100.1-10.40.199.254
/ip dhcp-server
add address-pool=dhcp_StaffMGMT  interface=vlan10_Staff lease-time=1d name=dhcpStaff
add address-pool=dhcp_Guest interface=vlan20_Guest lease-time=1d name=dhcpGuest
add address-pool=dhcp_VOIP interfacevlan30_VOIP lease-time=1w3d name=dhcpVOIP
add address-pool=dhcp_CCTV interface=vlan40_CCTV ease-time=1d name=dhcpCCTV
/interface bridge port
add bridge=bridge1 ingress-filtering=yes frame-types=admit-only-vlan-tagged interface=ether6
add bridge=bridge1 ingress-filtering=yes frame-types=admit-only-vlan-tagged interface=ether7
add bridge=bridge1 ingress-filtering=yes frame-types=admit-only-vlan-tagged interface=ether8
add bridge=bridge1 ingress-filtering=yes frame-types=admit-only-vlan-tagged interface=ether9
add bridge=bridge1 ingress-filtering=yes frame-types=admit-only-vlan-tagged interface=ether10
add bridge=bridge1 ingress-filtering=yes frame-typyes=admin-priority-and-untagged interface=ether11 pvid=10
add bridge=bridge1 ingress-filtering=yes frame-typyes=admin-priority-and-untagged interface=ether12 pvid=10
add bridge=bridge1 ingress-filtering=yes frame-typyes=admin-priority-and-untagged interface=ether13 pvid=10
/ip neighbor discovery-settings
set discover-interface-list=MGMT
/interface bridge vlan
add bridge=bridge1 tagged=bridge1,ether6,ether7,ether8,ether9,ether10  untagged=ether11,ether12,ether13  vlan-ids=10
add bridge=bridge1 tagged=bridge1,ether6,ether7,ether8,ether9,ether10  vlan-ids=20,30,40
/interface list members
add interface=WAN1 list=WAN
add interface=WAN2 list=WAN
add interface=vlan10_Staff list=LAN
add interface=vlan20_Guest list=LAN
add interface=vlan30_VOIP list=LAN
add interface=vlan40_CCTV list=LAN
add interface=vlan10_Staff list=MGMT
add interface=Off-Bridge5 list=MGMT
/ip address
add address=10.10.0.1/16 interface=vlan10_Staff network=10.10.0.0
add address=10.20.0.1/16 interface=vlan20_Guest network=10.20.0.0
add address=10.30.0.1/16 interface=vlan30_VOIP network=10.30.0.0
add address=10.40.0.1/16 interface=vlan40_CCTV network=10.40.0.0
add address=192.168.55.1/24	interface=Off-Bridge5 network=192.168.55.0
/ip dhcp-client
add add-default-route=no disabled=no interface=WAN1
add add-default-route=no disabled=no interface=WAN2
/ip dhcp-server network
add address=10.10.0.0/16 dns-server=10.10.0.1 gateway=10.10.0.1
add address=10.20.0.0/16 dns-server=10.20.0.1 gateway=10.20.0.1
add address=10.30.0.0/16 dns-server=10.30.0.1 gateway=10.30.0.1
add address=10.40.0.0/16 dns-server=10.40.0.1 gateway=10.40.0.1
/ip dns
set allow-remote-requests=yes servers=8.8.8.8,8.8.4.4
/ip firewall filter
add action=accept chain=input connection-state=established,related,untracked
add action=drop chain=input connection-state=invalid
add action=accept chain=input protocol=icmp
add action=accept chain=input comment=\
    "defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1	
add action=accept chain=input in-interface-list=MGMT
add action=accept chain=input comment="Users to DNS" dst-port=53 protocol=udp in-interface-list=LAN
add action=accept chain=input comment="Users to DNS" dst-port=53 protocol=tcp in-interface-list=LAN
add action=drop chain=input comment="Drop all else"
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
add action=fasttrack-connection chain=forward comment="fasttrack" \
    connection-state=established,related hw-offload=yes
add action=accept chain=forward comment="related-establ-untracked" \
    connection-state=established,related,untracked
add action=drop chain=forward connection-state=invalid
add action=accept chain=forward comment="allow internet traffic" \
    in-interface-list=LAN out-interface-list=WAN
add action=accept chain=forward comment="port forwarding" disabled=yes \
    connection-nat-state=dstnat
add action=accept chain=forward comment="MGMT to all vlans" \
     in-interface-list=MGMT out-interface-list=LAN
add action=drop chain=forward comment="drop all else"
/ip firewall nat
add action=masquerade chain=srcnat comment="defconf: masquerade" \
    ipsec-policy=out,none out-interface-list=WAN
/ip firewall mangle
add action=mark-connection chain=forward connection-mark=no-mark dst-address-type=!local \
    in-interface-list=LAN new-connection-mark=viaWAN1 passthrough=yes \
  	per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=forward connection-mark=no-mark dst-address-type=!local \
    in-interface-list=LAN new-connection-mark=viaWAN2 passthrough=yes \
    per-connection-classifier=both-addresses:2/1
add action=mark-routing chain=prerouting connection-mark=viaWAN1 \
    new-routing-mark=useWAN1 passthough=no
add action=mark-routing chain=prerouting connection-mark=viaWAN2 \
    new-routing-mark=useWAN2 passthough=no
/ip route
add check-gateway=ping dst-address=0.0.0.0/0 gateway=ISP1-gateway-IP routing-table=main
add distance=2 dst-address=0.0.0.0/0 gateway=ISP2-gateway-IP  routing-table=main
add dst-address=0.0.0.0/0 gateway=ISP1-gateway-IP  routing-table=useWAN1
add dst-address=0.0.0.0/0 gateway=ISP2-gateway-IP  routing-table=useWAN2
/tool mac-server
set allowed-interface-list=NONE
/tool mac-server mac-winbox
set allowed-interface-list=MGMT
/system clock
set time-zone-name=Europe/London

NetworqAndy · Mon Apr 15, 2024 12:41 pm

@anav:
Thanks for your reply. In answer to (some of) your questions, VLAN90 will eventually be another WAN connection from elsewhere on site. It's likely only WANs 1-3 will be local to the router.

For some reason the DHCP servers for the two orphaned pools had removed themselves from the exported configuration, but in winbox were showing with errors. I've recreated those servers this morning.

As per working through with thecat, VLAN10 is now being used for Staff and management. That was an oversight on my part leaving all the management side on VLAN1 (ie not on a VLAN).

I have discovered something very odd this morning though - if I disconnect WAN1 and WAN2, I can then ping the router (10.10.0.1) from any of the staff ports and things come back to life. As soon as WAN1 and/or WAN2 are reconnected I can no longer ping the router or winbox in via IP.... not sure if this highlights where the issue is?

***UPDATE:
I've been looking to see what's related to this - the issue with losing IP access/ability to ping is directly related to the route entry

/ip route
add check-gateway=ping distance=1 gateway=WAN1 routing-mark=viaWAN1

If I disable this static route, I regain access to the router.

NetworqAndy · Mon Apr 15, 2024 12:57 pm

Okay, even more specifically the issue is the routing-mark attribute within that rule. By removing that attribute, not only can I ping the router but machines also are able to ping/access the internet. Obviously I know this isn't the fix, but seem to have narrowed down where the issue might actually be lurking.

jaclaz · Mon Apr 15, 2024 1:56 pm

So, it derives from the mangle rules?

/ip firewall mangle
add action=mark-connection chain=forward connection-mark=no-mark dst-address-type=!local \
in-interface-list=LAN new-connection-mark=viaWAN1 passthrough=yes \
per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=forward connection-mark=no-mark dst-address-type=!local \
in-interface-list=LAN new-connection-mark=viaWAN2 passthrough=yes \
per-connection-classifier=both-addresses:2/1
add action=mark-routing chain=prerouting connection-mark=viaWAN1 \
new-routing-mark=useWAN1 passthough=no
add action=mark-routing chain=prerouting connection-mark=viaWAN2 \
new-routing-mark=useWAN2 passthough=no

It seems like you have a connection mark of "viaWAN1" BUT a routing mark of "useWAN1"

Maybe you should have:
/ip route
add check-gateway=ping distance=1 gateway=WAN1 routing-mark=useWAN1

?

anav · Mon Apr 15, 2024 2:13 pm

Sorry cannot comment without seeing latest config.

NetworqAndy · Mon Apr 15, 2024 5:03 pm

@anav - I found no change by changing over to the four mangle rules so have reverted back for now. Currently the two routes which specify a routing mark of viaWAN1 or viaWAN2 are what's preventing access to the router. Disabling them or removing the routing-mark attribute brings everything back to life

Latest Config:

# apr/15/2024 13:59:49 by RouterOS 6.49.13
# software id = E75X-80RJ
#
# model = RB1100Dx4
# serial number = HEY09AX5MMT
/interface bridge
add name=bridge1 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether1 ] name=WAN1
set [ find default-name=ether2 ] name=WAN2
/interface vlan
add interface=bridge1 name=vlan10_StaffMGMT vlan-id=10
add interface=bridge1 name=vlan20_Guest vlan-id=20
add interface=bridge1 name=vlan30_VOIP vlan-id=30
add interface=bridge1 name=vlan40_CCTV vlan-id=40
add interface=bridge1 name=vlan90_WAN3 vlan-id=90
/caps-man datapath
add bridge=bridge1 name=datapath_StaffMGMT vlan-id=10 vlan-mode=use-tag
add bridge=bridge1 name=datapath_Guest vlan-id=20 vlan-mode=use-tag
/caps-man security
add authentication-types=wpa2-psk encryption=aes-ccm name=security_StaffMGMT
add authentication-types=wpa2-psk encryption=aes-ccm name=security_Guest
/caps-man configuration
add country="united kingdom" datapath=datapath_Guest \
datapath.client-to-client-forwarding=no datapath.vlan-id=20 \
datapath.vlan-mode=use-tag installation=indoor mode=ap name=cfg_GuestWifi \
security=security_Guest ssid=OldMill_GuestWiFi
add country="united kingdom" datapath=datapath_StaffMGMT datapath.bridge=\
bridge1 installation=indoor mode=ap name=cfg_StaffMGMT security=\
security_StaffMGMT ssid=OldMill_Staff
/interface ethernet switch port
set 0 default-vlan-id=0
set 1 default-vlan-id=0
set 2 default-vlan-id=0
set 3 default-vlan-id=0
set 4 default-vlan-id=0
set 5 default-vlan-id=0
set 6 default-vlan-id=0
set 7 default-vlan-id=0
set 8 default-vlan-id=0
set 9 default-vlan-id=0
set 10 default-vlan-id=0
set 11 default-vlan-id=0
set 12 default-vlan-id=0
set 13 default-vlan-id=0
set 14 default-vlan-id=0
set 15 default-vlan-id=0
/interface list
add name=all_LANs
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=dhcp_StaffMGMT ranges=10.10.100.1-10.10.199.254
add name=dhcp_Guest ranges=10.20.100.1-10.20.199.254
add name=dhcp_VOIP ranges=10.30.100.1-10.30.199.254
add name=dhcp_CCTV ranges=10.40.100.1-10.40.199.254
/ip dhcp-server
add address-pool=dhcp_StaffMGMT disabled=no interface=vlan10_StaffMGMT \
lease-time=4w2d name=dhcpStaffMGMT
add address-pool=dhcp_Guest disabled=no interface=vlan20_Guest lease-time=1d \
name=dhcpGuest
add address-pool=dhcp_VOIP disabled=no interface=vlan30_VOIP lease-time=\
4w2d10m name=dhcpVOIP
add address-pool=dhcp_VOIP disabled=no interface=vlan40_CCTV lease-time=\
4w2d10m name=dhcpCCTV
/caps-man manager
set enabled=yes package-path=/ upgrade-policy=suggest-same-version
/caps-man manager interface
set [ find default=yes ] forbid=yes
add disabled=no interface=bridge1
/dude
set enabled=yes
/interface bridge port
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether5
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether6
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether7
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether8
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether9
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether10
add bridge=bridge1 interface=ether11 pvid=10
add bridge=bridge1 interface=ether12 pvid=10
add bridge=bridge1 interface=ether13 pvid=10
/interface bridge vlan
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9 \
untagged=ether10 vlan-ids=10
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9 \
untagged=ether10 vlan-ids=20
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9 \
untagged=ether10 vlan-ids=30
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9 \
untagged=ether10 vlan-ids=40
add bridge=bridge1 tagged=bridge1,ether5,ether6,ether7,ether8,ether9 \
untagged=ether10 vlan-ids=90
/interface list member
add interface=vlan10_StaffMGMT list=all_LANs
add interface=vlan20_Guest list=all_LANs
add interface=vlan30_VOIP list=all_LANs
add interface=vlan40_CCTV list=all_LANs
add interface=bridge1 list=all_LANs
/ip address
add address=10.30.0.1/16 interface=vlan30_VOIP network=10.30.0.0
add address=10.40.0.1/16 interface=vlan40_CCTV network=10.40.0.0
add address=10.10.0.1/16 interface=vlan10_StaffMGMT network=10.10.0.0
add address=10.20.0.1/16 interface=vlan20_Guest network=10.20.0.0
/ip dhcp-client
add add-default-route=no disabled=no interface=WAN1
add add-default-route=no disabled=no interface=WAN2
/ip dhcp-server lease
add address=10.10.0.11 mac-address=78:9A:18:2D:87:60 server=dhcpStaffMGMT
add address=10.10.0.101 client-id=1:d4:1:c3:57:47:2e mac-address=\
D4:01:C3:57:47:2E server=dhcpStaffMGMT
add address=10.10.0.102 client-id=1:d4:1:c3:57:20:3f mac-address=\
D4:01:C3:57:20:3F server=dhcpStaffMGMT
/ip dhcp-server network
add address=10.10.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.10.0.1
add address=10.20.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.20.0.1
add address=10.30.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.30.0.1
add address=10.40.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.40.0.1
/ip dns
set servers=8.8.8.8,8.8.4.4
/ip firewall address-list
add address=10.20.100.1-10.20.199.254 list=Guest
add address=10.10.100.0/24 list=local
add address=10.20.100.0/24 list=local
add address=10.30.100.0/24 list=local
add address=10.40.100.0/24 list=local
/ip firewall filter
add action=drop chain=input comment=\
"Block guest access to router local ports" dst-address=10.20.0.0/16 \
dst-port=80,21,22,23,8291 protocol=tcp src-address-list=Guest
add action=drop chain=input dst-address=10.10.0.0/16 src-address-list=Guest
add action=drop chain=input dst-address=10.30.0.0/16 src-address-list=Guest
add action=drop chain=input dst-address=10.40.0.0/16 src-address-list=Guest
add action=fasttrack-connection chain=forward connection-state=\
established,related
add action=accept chain=input connection-state=established,related
add action=accept chain=forward connection-state=established,related
add action=accept chain=input protocol=icmp
/ip firewall mangle
add action=mark-connection chain=prerouting connection-mark=no-mark \
in-interface=WAN1 new-connection-mark=viaWAN1 passthrough=yes
add action=mark-connection chain=prerouting connection-mark=no-mark \
in-interface=WAN2 new-connection-mark=viaWAN2 passthrough=yes
add action=mark-routing chain=prerouting connection-mark=viaWAN1 \
new-routing-mark=viaWAN1 passthrough=yes
add action=mark-routing chain=output connection-mark=viaWAN1 \
new-routing-mark=viaWAN1 passthrough=no
add action=mark-routing chain=prerouting connection-mark=viaWAN2 \
new-routing-mark=viaWAN2 passthrough=yes
add action=mark-routing chain=output connection-mark=viaWAN2 \
new-routing-mark=viaWAN2 passthrough=no
add action=mark-connection chain=prerouting connection-mark=no-mark \
dst-address-list=!local dst-address-type="" in-interface-list=all_LANs \
new-connection-mark=viaWAN1 passthrough=yes per-connection-classifier=\
both-addresses:2/0
add action=mark-connection chain=prerouting connection-mark=no-mark \
dst-address-list=!local dst-address-type="" in-interface-list=all_LANs \
new-connection-mark=viaWAN2 passthrough=yes per-connection-classifier=\
both-addresses:2/1
add action=fasttrack-connection chain=forward connection-mark=no-mark \
connection-state=established,related
/ip firewall nat
add action=masquerade chain=srcnat out-interface=WAN1
add action=masquerade chain=srcnat out-interface=WAN2
add action=masquerade chain=srcnat disabled=yes out-interface=WAN1 \
src-address=10.10.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN2 \
src-address=10.10.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN1 \
src-address=10.20.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN2 \
src-address=10.20.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN1 \
src-address=10.30.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN2 \
src-address=10.30.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN1 \
src-address=10.40.0.0/16
add action=masquerade chain=srcnat disabled=yes out-interface=WAN2 \
src-address=10.40.0.0/16
/ip route
add check-gateway=ping distance=1 gateway=WAN1 routing-mark=viaWAN1
add distance=2 gateway=WAN2 routing-mark=viaWAN2
add check-gateway=ping distance=1 gateway=WAN1
add check-gateway=ping distance=1 gateway=WAN2
/ip route rule
add table=viaWAN1
add table=viaWAN2
add table=viaWAN3
/system clock
set time-zone-name=Europe/London
/system identity
set name=RB1100-Reception

anav · Mon Apr 15, 2024 5:23 pm

I provided a clean clear config, which you ignored.
Thus unable to help further. If you had done all that was asked not just part of the mangles, then we could make progress.
Not saying it would be 100%, but we could narrow further the problem areas with some certainty.
The config has too many spots with issues to try and deal in onsies twosies.
Gluck!

NetworqAndy · Mon Apr 15, 2024 6:37 pm

Hi anav,

Apologies - I posted the older version of the code.

Here's the version after I tried your changes. Exactly the same behaviours - if either of the WAN interfaces are connected, you lose the ability to ping or winbox in via IP.

With your suggested config I can't access the internet at all.

# apr/15/2024 15:35:04 by RouterOS 6.49.13
# software id = E75X-80RJ
#
# model = RB1100Dx4
# serial number = HEY09AX5MMT
/interface bridge
add name=bridge1 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether5 ] name=OffBridge-5
set [ find default-name=ether1 ] name=WAN1
set [ find default-name=ether2 ] name=WAN2
set [ find default-name=ether3 ] name=WAN3-Future
set [ find default-name=ether4 ] name=WAN4-Future
/interface vlan
add interface=bridge1 name=vlan10_StaffMGMT vlan-id=10
add interface=bridge1 name=vlan20_Guest vlan-id=20
add interface=bridge1 name=vlan30_VOIP vlan-id=30
add interface=bridge1 name=vlan40_CCTV vlan-id=40
add comment=FutureWAN interface=bridge1 name=vlan90_WAN3 vlan-id=90
/caps-man datapath
add bridge=bridge1 name=datapath_StaffMGMT vlan-id=10 vlan-mode=use-tag
add bridge=bridge1 name=datapath_Guest vlan-id=20 vlan-mode=use-tag
/caps-man security
add authentication-types=wpa2-psk encryption=aes-ccm name=security_StaffMGMT
add authentication-types=wpa2-psk encryption=aes-ccm name=security_Guest
/caps-man configuration
add country="united kingdom" datapath=datapath_Guest \
    datapath.client-to-client-forwarding=no datapath.vlan-id=20 \
    datapath.vlan-mode=use-tag installation=indoor mode=ap name=cfg_GuestWifi \
    security=security_Guest ssid=OldMill_GuestWiFi
add country="united kingdom" datapath=datapath_StaffMGMT datapath.bridge=\
    bridge1 installation=indoor mode=ap name=cfg_StaffMGMT security=\
    security_StaffMGMT ssid=OldMill_Staff
/interface ethernet switch port
set 0 default-vlan-id=0
set 1 default-vlan-id=0
set 2 default-vlan-id=0
set 3 default-vlan-id=0
set 4 default-vlan-id=0
set 5 default-vlan-id=0
set 6 default-vlan-id=0
set 7 default-vlan-id=0
set 8 default-vlan-id=0
set 9 default-vlan-id=0
set 10 default-vlan-id=0
set 11 default-vlan-id=0
set 12 default-vlan-id=0
set 13 default-vlan-id=0
set 14 default-vlan-id=0
set 15 default-vlan-id=0
/interface list
add name=WAN
add name=LAN
add name=MGMT
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=dhcp_StaffMGMT ranges=10.10.100.1-10.10.199.254
add name=dhcp_Guest ranges=10.20.100.1-10.20.199.254
add name=dhcp_VOIP ranges=10.30.100.1-10.30.199.254
add name=dhcp_CCTV ranges=10.40.100.1-10.40.199.254
/ip dhcp-server
add address-pool=dhcp_StaffMGMT disabled=no interface=vlan10_StaffMGMT \
    lease-time=4w2d name=dhcpStaffMGMT
add address-pool=dhcp_Guest disabled=no interface=vlan20_Guest lease-time=1d \
    name=dhcpGuest
add address-pool=dhcp_VOIP disabled=no interface=vlan30_VOIP lease-time=\
    4w2d10m name=dhcpVOIP
add address-pool=dhcp_VOIP disabled=no interface=vlan40_CCTV lease-time=\
    4w2d10m name=dhcpCCTV
/caps-man manager
set enabled=yes package-path=/ upgrade-policy=suggest-same-version
/caps-man manager interface
set [ find default=yes ] forbid=yes
add disabled=no interface=bridge1
/dude
set enabled=yes
/interface bridge port
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether6
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether7
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether8
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether9
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether10
add bridge=bridge1 frame-types=admit-only-untagged-and-priority-tagged \
    ingress-filtering=yes interface=ether11 pvid=10
add bridge=bridge1 ingress-filtering=yes interface=ether12 pvid=10
add bridge=bridge1 ingress-filtering=yes interface=ether13 pvid=10
/ip neighbor discovery-settings
set discover-interface-list=MGMT
/interface bridge vlan
add bridge=bridge1 tagged=bridge1,ether6,ether7,ether8,ether9,ether10 \
    untagged=ether11,ether12,ether13 vlan-ids=10
add bridge=bridge1 tagged=bridge1,ether6,ether7,ether8,ether9,ether10 \
    vlan-ids=20,30,40
/interface list member
add interface=WAN1 list=WAN
add interface=WAN2 list=WAN
add interface=vlan10_StaffMGMT list=LAN
add interface=vlan20_Guest list=LAN
add interface=vlan30_VOIP list=LAN
add interface=vlan40_CCTV list=LAN
add interface=vlan10_StaffMGMT list=MGMT
add interface=OffBridge-5 list=MGMT
/ip address
add address=10.30.0.1/16 interface=vlan30_VOIP network=10.30.0.0
add address=10.40.0.1/16 interface=vlan40_CCTV network=10.40.0.0
add address=10.10.0.1/16 interface=vlan10_StaffMGMT network=10.10.0.0
add address=10.20.0.1/16 interface=vlan20_Guest network=10.20.0.0
add address=192.168.55.1/24 interface=OffBridge-5 network=192.168.55.0
/ip dhcp-client
add add-default-route=no disabled=no interface=WAN1
add add-default-route=no disabled=no interface=WAN2
/ip dhcp-server lease
add address=10.10.0.11 mac-address=78:9A:18:2D:87:60 server=dhcpStaffMGMT
add address=10.10.0.101 client-id=1:d4:1:c3:57:47:2e mac-address=\
    D4:01:C3:57:47:2E server=dhcpStaffMGMT
add address=10.10.0.102 client-id=1:d4:1:c3:57:20:3f mac-address=\
    D4:01:C3:57:20:3F server=dhcpStaffMGMT
/ip dhcp-server network
add address=10.10.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.10.0.1
add address=10.20.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.20.0.1
add address=10.30.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.30.0.1
add address=10.40.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.40.0.1
/ip dns
set servers=8.8.8.8,8.8.4.4
/ip firewall address-list
add address=10.20.100.1-10.20.199.254 list=Guest
add address=10.10.100.0/24 list=local
add address=10.20.100.0/24 list=local
add address=10.30.100.0/24 list=local
add address=10.40.100.0/24 list=local
/ip firewall filter
add action=drop chain=input comment=\
    "Block guest access to router local ports" dst-address=10.20.0.0/16 \
    dst-port=80,21,22,23,8291 protocol=tcp src-address-list=Guest
add action=drop chain=input dst-address=10.10.0.0/16 src-address-list=Guest
add action=drop chain=input dst-address=10.30.0.0/16 src-address-list=Guest
add action=drop chain=input dst-address=10.40.0.0/16 src-address-list=Guest
add action=accept chain=input comment=fasttrack connection-state=\
    established,related,untracked
add action=drop chain=forward connection-state=invalid
add action=accept chain=input protocol=icmp
add action=accept chain=input comment=\
    "defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=accept chain=input in-interface-list=MGMT
add action=accept chain=input comment="Users to DNS" dst-port=53 \
    in-interface-list=LAN protocol=udp
add action=accept chain=input comment="Users to DNS" dst-port=53 \
    in-interface-list=LAN protocol=tcp
add action=drop chain=input comment="Drop all else"
add action=fasttrack-connection chain=forward comment=fasttrack \
    connection-state=established,related
add action=accept chain=forward comment=related-established-untracked \
    connection-state=established,related,untracked
add action=drop chain=forward connection-state=invalid
add action=accept chain=forward comment="allow internet traffic" \
    in-interface-list=LAN out-interface-list=WAN
add action=accept chain=forward comment="port forwarding" \
    connection-nat-state=dstnat disabled=yes
add action=accept chain=forward comment="MGMT to all vlans" \
    in-interface-list=MGMT out-interface-list=LAN
add action=drop chain=forward comment="drop all else"
/ip firewall mangle
add action=mark-connection chain=forward connection-mark=no-mark \
    dst-address-type=!local in-interface-list=LAN new-connection-mark=viaWAN1 \
    passthrough=yes per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=forward connection-mark=no-mark \
    dst-address-type=!local in-interface-list=LAN new-connection-mark=viaWAN2 \
    passthrough=yes per-connection-classifier=both-addresses:2/1
add action=mark-routing chain=prerouting connection-mark=viaWAN1 \
    new-routing-mark=useWAN1 passthrough=no
add action=mark-routing chain=prerouting connection-mark=viaWAN2 \
    new-routing-mark=useWAN2 passthrough=no
/ip firewall nat
add action=accept chain=srcnat comment="defconf: masquerade" ipsec-policy=\
    out,none out-interface-list=WAN
/ip route
add distance=1 gateway=192.168.2.1 routing-mark=useWAN1
add distance=1 gateway=192.168.3.1 routing-mark=useWAN2
add check-gateway=ping distance=1 gateway=192.168.2.1
add distance=2 gateway=192.168.3.1
/ip route rule
add table=useWAN1
add table=useWAN2
/system clock
set time-zone-name=Europe/London
/system identity
set name=RB1100-Reception
/tool mac-server
set allowed-interface-list=none
/tool mac-server mac-winbox
set allowed-interface-list=MGMT

anav · Mon Apr 15, 2024 7:36 pm

UPDATE JUST REALIZED THAT THIS IS VERSION SIX, So need to adjust ROUTES etc...... So pay close attention to 4.5.6. etc......
(1) Okay to be clear, you can winbox in no problem its just using winbox via IP that is not working? ( or webconfig I suppose ).

(2) WAN does not go on bridge!
add comment=FutureWAN interface=bridge1 name=vlan90_WAN3 vlan-id=90
instead
/interface vlan
add comment=futurewan interface=WAN3-Future name=vlan90 vlan-id=90

(3) Your firewall rules are nonsensical, I will give it one more try. To be clear in this config, the guest network has no access to the router for config and has no access to any other vlans.
Such made up firewalls without any knowledge will have weird effects, so make it clean. If you dont understand something ask!

/ip firewall filter
add action=accept chain=input connection-state=established,related,untracked
 add action=drop chain=input connection-state=invalid
add action=accept chain=input protocol=icmp
add action=accept chain=input comment=\
    "defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1	
add action=accept chain=input in-interface-list=MGMT
add action=accept chain=input comment="Users to DNS" dst-port=53 protocol=udp in-interface-list=LAN
add action=accept chain=input comment="Users to DNS" dst-port=53 protocol=tcp in-interface-list=LAN
add action=drop chain=input comment="Drop all else"
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
add action=fasttrack-connection chain=forward comment="fasttrack" \
    connection-state=established,related hw-offload=yes
add action=accept chain=forward comment="related-establ-untracked" \
    connection-state=established,related,untracked
add action=drop chain=forward connection-state=invalid
add action=accept chain=forward comment="allow internet traffic" \
    in-interface-list=LAN out-interface-list=WAN
add action=accept chain=forward comment="port forwarding" disabled=yes \
    connection-nat-state=dstnat
add action=accept chain=forward comment="MGMT to all vlans" \
     in-interface-list=MGMT out-interface-list=LAN
add action=drop chain=forward comment="drop all else"

...
4. Okay since this is version6, Routing Tables are NOT created.

5. Okay I see you have the two main table routes, which are mandatory to have on the router!! Be it version 6 or 7.

6. However, we need to ensure the IP routes are formatted properly for version 6.

/ip route
add check-gateway=ping dst-address=0.0.0.0/0 gateway=ISP1-gateway-IP routing-table=main
add distance=2 dst-address=0.0.0.0/0 gateway=ISP2-gateway-IP routing-table=main
add dst-address=0.0.0.0/0 gateway=ISP1-gateway-IP routing-mark=useWAN1
add dst-address=0.0.0.0/0 gateway=ISP2-gateway-IP routing-mark=useWAN2

NetworqAndy · Tue Apr 16, 2024 12:33 pm

Hi Anav,

Thanks for that - so I've gone through and recreated all the firewall filter rules as per your script. Only comment there would be on the fasttrack rule as hw-offload is not a valid command name.

In terms of the routes, the first two entries contain "routing-table" - again creating these two rules with that argument in makes for an invalid command name error. My understanding in ROS6 is that you can only use routing-mark arguments? However if I use routing-mark=main, it's effectively a null argument as the default table is main anyway.

Current config is below. Net result is still exactly the same problem - as soon as either WAN1 or WAN2 are connected/enabled, I lose the ability to even ping the router. Winbox access is only possible via MAC address.

Andy

# apr/16/2024 10:32:07 by RouterOS 6.49.13
# software id = E75X-80RJ
#
# model = RB1100Dx4
# serial number = HEY09AX5MMT
/interface bridge
add name=bridge1 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether5 ] name=OffBridge-5
set [ find default-name=ether1 ] name=WAN1
set [ find default-name=ether2 ] name=WAN2
set [ find default-name=ether3 ] name=WAN3-Future
set [ find default-name=ether4 ] name=WAN4-Future
/interface vlan
add interface=bridge1 name=vlan10_StaffMGMT vlan-id=10
add interface=bridge1 name=vlan20_Guest vlan-id=20
add interface=bridge1 name=vlan30_VOIP vlan-id=30
add interface=bridge1 name=vlan40_CCTV vlan-id=40
add comment=FutureWAN interface=WAN3-Future name=vlan90_WAN3 vlan-id=90
/caps-man datapath
add bridge=bridge1 name=datapath_StaffMGMT vlan-id=10 vlan-mode=use-tag
add bridge=bridge1 name=datapath_Guest vlan-id=20 vlan-mode=use-tag
/caps-man security
add authentication-types=wpa2-psk encryption=aes-ccm name=security_StaffMGMT
add authentication-types=wpa2-psk encryption=aes-ccm name=security_Guest
/caps-man configuration
add country="united kingdom" datapath=datapath_Guest \
    datapath.client-to-client-forwarding=no datapath.vlan-id=20 \
    datapath.vlan-mode=use-tag installation=indoor mode=ap name=cfg_GuestWifi \
    security=security_Guest ssid=OldMill_GuestWiFi
add country="united kingdom" datapath=datapath_StaffMGMT datapath.bridge=\
    bridge1 installation=indoor mode=ap name=cfg_StaffMGMT security=\
    security_StaffMGMT ssid=OldMill_Staff
/interface ethernet switch port
set 0 default-vlan-id=0
set 1 default-vlan-id=0
set 2 default-vlan-id=0
set 3 default-vlan-id=0
set 4 default-vlan-id=0
set 5 default-vlan-id=0
set 6 default-vlan-id=0
set 7 default-vlan-id=0
set 8 default-vlan-id=0
set 9 default-vlan-id=0
set 10 default-vlan-id=0
set 11 default-vlan-id=0
set 12 default-vlan-id=0
set 13 default-vlan-id=0
set 14 default-vlan-id=0
set 15 default-vlan-id=0
/interface list
add name=WAN
add name=LAN
add name=MGMT
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=dhcp_StaffMGMT ranges=10.10.100.1-10.10.199.254
add name=dhcp_Guest ranges=10.20.100.1-10.20.199.254
add name=dhcp_VOIP ranges=10.30.100.1-10.30.199.254
add name=dhcp_CCTV ranges=10.40.100.1-10.40.199.254
/ip dhcp-server
add address-pool=dhcp_StaffMGMT disabled=no interface=vlan10_StaffMGMT \
    lease-time=4w2d name=dhcpStaffMGMT
add address-pool=dhcp_Guest disabled=no interface=vlan20_Guest lease-time=1d \
    name=dhcpGuest
add address-pool=dhcp_VOIP disabled=no interface=vlan30_VOIP lease-time=\
    4w2d10m name=dhcpVOIP
add address-pool=dhcp_VOIP disabled=no interface=vlan40_CCTV lease-time=\
    4w2d10m name=dhcpCCTV
/caps-man manager
set enabled=yes package-path=/ upgrade-policy=suggest-same-version
/caps-man manager interface
set [ find default=yes ] forbid=yes
add disabled=no interface=bridge1
/dude
set enabled=yes
/interface bridge port
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether6
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether7
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether8
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether9
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether10
add bridge=bridge1 frame-types=admit-only-untagged-and-priority-tagged \
    ingress-filtering=yes interface=ether11 pvid=10
add bridge=bridge1 ingress-filtering=yes interface=ether12 pvid=10
add bridge=bridge1 ingress-filtering=yes interface=ether13 pvid=10
/ip neighbor discovery-settings
set discover-interface-list=MGMT
/interface bridge vlan
add bridge=bridge1 tagged=bridge1,ether6,ether7,ether8,ether9,ether10 \
    untagged=ether11,ether12,ether13 vlan-ids=10
add bridge=bridge1 tagged=bridge1,ether6,ether7,ether8,ether9,ether10 \
    vlan-ids=20,30,40
/interface list member
add interface=WAN1 list=WAN
add interface=WAN2 list=WAN
add interface=vlan10_StaffMGMT list=LAN
add interface=vlan20_Guest list=LAN
add interface=vlan30_VOIP list=LAN
add interface=vlan40_CCTV list=LAN
add interface=vlan10_StaffMGMT list=MGMT
add interface=OffBridge-5 list=MGMT
/ip address
add address=10.30.0.1/16 interface=vlan30_VOIP network=10.30.0.0
add address=10.40.0.1/16 interface=vlan40_CCTV network=10.40.0.0
add address=10.10.0.1/16 interface=vlan10_StaffMGMT network=10.10.0.0
add address=10.20.0.1/16 interface=vlan20_Guest network=10.20.0.0
add address=192.168.55.1/24 interface=OffBridge-5 network=192.168.55.0
/ip dhcp-client
add add-default-route=no disabled=no interface=WAN1
add add-default-route=no disabled=no interface=WAN2
/ip dhcp-server lease
add address=10.10.0.11 mac-address=78:9A:18:2D:87:60 server=dhcpStaffMGMT
add address=10.10.0.101 client-id=1:d4:1:c3:57:47:2e mac-address=\
    D4:01:C3:57:47:2E server=dhcpStaffMGMT
add address=10.10.0.102 client-id=1:d4:1:c3:57:20:3f mac-address=\
    D4:01:C3:57:20:3F server=dhcpStaffMGMT
/ip dhcp-server network
add address=10.10.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.10.0.1
add address=10.20.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.20.0.1
add address=10.30.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.30.0.1
add address=10.40.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.40.0.1
/ip dns
set servers=8.8.8.8,8.8.4.4
/ip firewall address-list
add address=10.20.100.1-10.20.199.254 list=Guest
add address=10.10.100.0/24 list=local
add address=10.20.100.0/24 list=local
add address=10.30.100.0/24 list=local
add address=10.40.100.0/24 list=local
/ip firewall filter
add action=accept chain=input connection-state=established,related,untracked
add action=drop chain=input connection-state=invalid
add action=accept chain=input protocol=icmp
add action=accept chain=input comment=\
    "defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=accept chain=input in-interface-list=MGMT
add action=accept chain=input dst-port=53 in-interface-list=LAN protocol=udp
add action=accept chain=input dst-port=53 in-interface-list=LAN protocol=tcp
add action=drop chain=input comment="Drop all else"
add action=fasttrack-connection chain=forward comment=fasttrack \
    connection-state=established,related
add action=accept chain=forward comment=related-establ-untracked \
    connection-state=established,related,untracked
add action=drop chain=forward connection-state=invalid
add action=accept chain=forward comment="allow internet traffic" \
    in-interface-list=LAN out-interface-list=WAN
add action=accept chain=forward comment="port forwarding" \
    connection-nat-state=dstnat disabled=yes
add action=accept chain=forward comment="MGMT to all vlans" \
    in-interface-list=MGMT out-interface-list=LAN
add action=drop chain=forward comment="drop all else"
/ip firewall mangle
add action=mark-connection chain=forward connection-mark=no-mark \
    dst-address-type=!local in-interface-list=LAN new-connection-mark=viaWAN1 \
    passthrough=yes per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=forward connection-mark=no-mark \
    dst-address-type=!local in-interface-list=LAN new-connection-mark=viaWAN2 \
    passthrough=yes per-connection-classifier=both-addresses:2/1
add action=mark-routing chain=prerouting connection-mark=viaWAN1 \
    new-routing-mark=useWAN1 passthrough=no
add action=mark-routing chain=prerouting connection-mark=viaWAN2 \
    new-routing-mark=useWAN2 passthrough=no
/ip firewall nat
add action=accept chain=srcnat comment="defconf: masquerade" ipsec-policy=\
    out,none out-interface-list=WAN
/ip route
add distance=1 gateway=192.168.2.1 routing-mark=useWAN1
add distance=1 gateway=192.168.3.1 routing-mark=useWAN2
add check-gateway=ping distance=1 gateway=192.168.2.1
add distance=2 gateway=192.168.3.1
/ip route rule
add table=useWAN1
add table=useWAN2
/system clock
set time-zone-name=Europe/London
/system identity
set name=RB1100-Reception

anav · Tue Apr 16, 2024 1:18 pm

Remove the bogus entries in Orange. Dont use input chain for rules pertaining to the forward chain, they are not needed anyway in the forward chain as we drop all other traffic.
There is no fastrack in input chain.
Add missing rule (in blue).

/ip firewall filter
add action=drop chain=input comment=\
"Block guest access to router local ports" dst-address=10.20.0.0/16 \
dst-port=80,21,22,23,8291 protocol=tcp src-address-list=Guest
add action=drop chain=input dst-address=10.10.0.0/16 src-address-list=Guest
add action=drop chain=input dst-address=10.30.0.0/16 src-address-list=Guest
add action=drop chain=input dst-address=10.40.0.0/16 src-address-list=Guest
-------------------------------------------------------------
add action=accept chain=input comment=fasttrack connection-state=\
established,related,untracked

add action=accept chain=input comment=related-established-untracked \
connection-state=established,related,untracked
add action=drop chain=forward connection-state=invalid
add action=accept chain=input protocol=icmp
add action=accept chain=input comment=\
"defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=accept chain=input in-interface-list=MGMT
add action=accept chain=input comment="Users to DNS" dst-port=53 \
in-interface-list=LAN protocol=udp
add action=accept chain=input comment="Users to DNS" dst-port=53 \
in-interface-list=LAN protocol=tcp
add action=drop chain=input comment="Drop all else"
add action=fasttrack-connection chain=forward comment=fasttrack \
connection-state=established,related
add action=accept chain=forward comment=related-established-untracked \
connection-state=established,related,untracked
add action=drop chain=forward connection-state=invalid
add action=accept chain=forward comment="allow internet traffic" \
in-interface-list=LAN out-interface-list=WAN
add action=accept chain=forward comment="port forwarding" \
connection-nat-state=dstnat disabled=yes
add action=accept chain=forward comment="MGMT to all vlans" \
in-interface-list=MGMT out-interface-list=LAN
add action=drop chain=forward comment="drop all else"

Remove the routing rules
/ip route rule
add table=useWAN1
add table=useWAN2

Please make the changes necessary and try again.
If its still not working then I suspect you are attempting to do this via wifi and capsman is getting in the way.

NetworqAndy · Tue Apr 16, 2024 1:35 pm

Hi Anav,

Apologies - I think you might just have opened the page before I edited the post. I managed to post yesterday's config again as it was the last thing I'd copied/pasted on this machine. I'd corrected it but clearly not before you opened the latest post!

Those entries in orange aren't in there, and the blue line is correct.

I have removed those tables however and I don't now lose the ability to ping the router.

The only remaining issues I don't however have access on via the WAN ports to the internet. I can ping using from within the router using WAN1 or WAN 2 so those are good, however there' still an issue lurking with the route. I can see packet counts incrementing on the srcnat entry, and torch shows outgoing packets destined for the server being pinged, but the returning ping isn't making it through.

One thing torch does highlight looking on the WAN side though (ie post NAT) is that the outbound packets are from a source IP of the laptop's IP address sitting on VLAN1. As such it's looking like an issue now within the masquerade/NAT config.

Just to be clear, config is now (with slight tweak on capsman config to allow for provisioning over VLAN10):

# apr/16/2024 11:37:53 by RouterOS 6.49.13
# software id = E75X-80RJ
#
# model = RB1100Dx4
# serial number = HEY09AX5MMT
/interface bridge
add name=bridge1 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether5 ] name=OffBridge-5
set [ find default-name=ether1 ] name=WAN1
set [ find default-name=ether2 ] name=WAN2
set [ find default-name=ether3 ] name=WAN3-Future
set [ find default-name=ether4 ] name=WAN4-Future
/caps-man interface
add disabled=no l2mtu=1600 mac-address=D4:01:C3:57:47:34 master-interface=\
    none name=cap1 radio-mac=D4:01:C3:57:47:34 radio-name=D401C3574734
add disabled=no l2mtu=1600 mac-address=D4:01:C3:57:47:33 master-interface=\
    none name=cap2 radio-mac=D4:01:C3:57:47:33 radio-name=D401C3574733
add disabled=no l2mtu=1600 mac-address=D4:01:C3:57:20:45 master-interface=\
    none name=cap3 radio-mac=D4:01:C3:57:20:45 radio-name=D401C3572045
add disabled=no l2mtu=1600 mac-address=D4:01:C3:57:20:44 master-interface=\
    none name=cap4 radio-mac=D4:01:C3:57:20:44 radio-name=D401C3572044
/interface vlan
add interface=bridge1 name=vlan10_StaffMGMT vlan-id=10
add interface=bridge1 name=vlan20_Guest vlan-id=20
add interface=bridge1 name=vlan30_VOIP vlan-id=30
add interface=bridge1 name=vlan40_CCTV vlan-id=40
add comment=FutureWAN interface=WAN3-Future name=vlan90_WAN3 vlan-id=90
/caps-man datapath
add bridge=bridge1 name=datapath_StaffMGMT vlan-id=10 vlan-mode=use-tag
add bridge=bridge1 name=datapath_Guest vlan-id=20 vlan-mode=use-tag
/caps-man security
add authentication-types=wpa2-psk encryption=aes-ccm name=security_StaffMGMT
add authentication-types=wpa2-psk encryption=aes-ccm name=security_Guest
/caps-man configuration
add country="united kingdom" datapath=datapath_Guest \
    datapath.client-to-client-forwarding=no datapath.vlan-id=20 \
    datapath.vlan-mode=use-tag installation=indoor mode=ap name=cfg_GuestWifi \
    security=security_Guest ssid=OldMill_GuestWiFi
add country="united kingdom" datapath=datapath_StaffMGMT datapath.bridge=\
    bridge1 installation=indoor mode=ap name=cfg_StaffMGMT security=\
    security_StaffMGMT ssid=OldMill_Staff
/interface ethernet switch port
set 0 default-vlan-id=0
set 1 default-vlan-id=0
set 2 default-vlan-id=0
set 3 default-vlan-id=0
set 4 default-vlan-id=0
set 5 default-vlan-id=0
set 6 default-vlan-id=0
set 7 default-vlan-id=0
set 8 default-vlan-id=0
set 9 default-vlan-id=0
set 10 default-vlan-id=0
set 11 default-vlan-id=0
set 12 default-vlan-id=0
set 13 default-vlan-id=0
set 14 default-vlan-id=0
set 15 default-vlan-id=0
/interface list
add name=WAN
add name=LAN
add name=MGMT
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=dhcp_StaffMGMT ranges=10.10.100.1-10.10.199.254
add name=dhcp_Guest ranges=10.20.100.1-10.20.199.254
add name=dhcp_VOIP ranges=10.30.100.1-10.30.199.254
add name=dhcp_CCTV ranges=10.40.100.1-10.40.199.254
/ip dhcp-server
add address-pool=dhcp_StaffMGMT disabled=no interface=vlan10_StaffMGMT \
    lease-time=4w2d name=dhcpStaffMGMT
add address-pool=dhcp_Guest disabled=no interface=vlan20_Guest lease-time=1d \
    name=dhcpGuest
add address-pool=dhcp_VOIP disabled=no interface=vlan30_VOIP lease-time=\
    4w2d10m name=dhcpVOIP
add address-pool=dhcp_VOIP disabled=no interface=vlan40_CCTV lease-time=\
    4w2d10m name=dhcpCCTV
/caps-man manager
set enabled=yes package-path=/ upgrade-policy=suggest-same-version
/caps-man manager interface
set [ find default=yes ] forbid=yes
add disabled=no interface=bridge1
add disabled=no interface=vlan10_StaffMGMT
/caps-man provisioning
add action=create-dynamic-enabled master-configuration=cfg_StaffMGMT \
    name-format=identity slave-configurations=cfg_GuestWifi
/dude
set enabled=yes
/interface bridge port
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether6
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether7
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether8
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether9
add bridge=bridge1 frame-types=admit-only-vlan-tagged ingress-filtering=yes \
    interface=ether10
add bridge=bridge1 frame-types=admit-only-untagged-and-priority-tagged \
    ingress-filtering=yes interface=ether11 pvid=10
add bridge=bridge1 ingress-filtering=yes interface=ether12 pvid=10
add bridge=bridge1 ingress-filtering=yes interface=ether13 pvid=10
/ip neighbor discovery-settings
set discover-interface-list=MGMT
/interface bridge vlan
add bridge=bridge1 tagged=bridge1,ether6,ether7,ether8,ether9,ether10 \
    untagged=ether11,ether12,ether13 vlan-ids=10
add bridge=bridge1 tagged=bridge1,ether6,ether7,ether8,ether9,ether10 \
    vlan-ids=20,30,40
/interface list member
add interface=WAN1 list=WAN
add interface=WAN2 list=WAN
add interface=vlan10_StaffMGMT list=LAN
add interface=vlan20_Guest list=LAN
add interface=vlan30_VOIP list=LAN
add interface=vlan40_CCTV list=LAN
add interface=vlan10_StaffMGMT list=MGMT
add interface=OffBridge-5 list=MGMT
/ip address
add address=10.30.0.1/16 interface=vlan30_VOIP network=10.30.0.0
add address=10.40.0.1/16 interface=vlan40_CCTV network=10.40.0.0
add address=10.10.0.1/16 interface=vlan10_StaffMGMT network=10.10.0.0
add address=10.20.0.1/16 interface=vlan20_Guest network=10.20.0.0
add address=192.168.55.1/24 interface=OffBridge-5 network=192.168.55.0
/ip dhcp-client
add add-default-route=no disabled=no interface=WAN1
add add-default-route=no disabled=no interface=WAN2
/ip dhcp-server lease
add address=10.10.0.11 mac-address=78:9A:18:2D:87:60 server=dhcpStaffMGMT
add address=10.10.0.101 client-id=1:d4:1:c3:57:47:2e mac-address=\
    D4:01:C3:57:47:2E server=dhcpStaffMGMT
add address=10.10.0.102 client-id=1:d4:1:c3:57:20:3f mac-address=\
    D4:01:C3:57:20:3F server=dhcpStaffMGMT
/ip dhcp-server network
add address=10.10.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.10.0.1
add address=10.20.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.20.0.1
add address=10.30.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.30.0.1
add address=10.40.0.0/16 dns-server=8.8.8.8,8.8.4.4 gateway=10.40.0.1
/ip dns
set servers=8.8.8.8,8.8.4.4
/ip firewall address-list
add address=10.20.100.1-10.20.199.254 list=Guest
add address=10.10.100.0/24 list=local
add address=10.20.100.0/24 list=local
add address=10.30.100.0/24 list=local
add address=10.40.100.0/24 list=local
/ip firewall filter
add action=accept chain=input connection-state=established,related,untracked
add action=drop chain=input connection-state=invalid
add action=accept chain=input protocol=icmp
add action=accept chain=input comment=\
    "defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=accept chain=input in-interface-list=MGMT
add action=accept chain=input dst-port=53 in-interface-list=LAN protocol=udp
add action=accept chain=input dst-port=53 in-interface-list=LAN protocol=tcp
add action=drop chain=input comment="Drop all else"
add action=fasttrack-connection chain=forward comment=fasttrack \
    connection-state=established,related
add action=accept chain=forward comment=related-establ-untracked \
    connection-state=established,related,untracked
add action=drop chain=forward connection-state=invalid
add action=accept chain=forward comment="allow internet traffic" \
    in-interface-list=LAN out-interface-list=WAN
add action=accept chain=forward comment="port forwarding" \
    connection-nat-state=dstnat disabled=yes
add action=accept chain=forward comment="MGMT to all vlans" \
    in-interface-list=MGMT out-interface-list=LAN
add action=drop chain=forward comment="drop all else"
/ip firewall mangle
add action=mark-connection chain=forward connection-mark=no-mark \
    dst-address-type=!local in-interface-list=LAN new-connection-mark=viaWAN1 \
    passthrough=yes per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=forward connection-mark=no-mark \
    dst-address-type=!local in-interface-list=LAN new-connection-mark=viaWAN2 \
    passthrough=yes per-connection-classifier=both-addresses:2/1
add action=mark-routing chain=prerouting connection-mark=viaWAN1 \
    new-routing-mark=useWAN1 passthrough=no
add action=mark-routing chain=prerouting connection-mark=viaWAN2 \
    new-routing-mark=useWAN2 passthrough=no
/ip firewall nat
add action=accept chain=srcnat comment="defconf: masquerade" ipsec-policy=\
    out,none out-interface-list=WAN
/ip route
add distance=1 gateway=192.168.2.1 routing-mark=useWAN1
add distance=1 gateway=192.168.3.1 routing-mark=useWAN2
add check-gateway=ping distance=1 gateway=192.168.2.1
add distance=2 gateway=192.168.3.1
/system clock
set time-zone-name=Europe/London
/system identity
set name=RB1100-Reception

NetworqAndy · Tue Apr 16, 2024 3:45 pm

FOUND IT!!!

The issue was the srcnat rule was set to accept rather than masquerade. So the two issues lurking here had been the presence of the two routing rule tables (which prevented any layer 3 access to the router as soon as a WAN port was active) and the config issue on the srcnat rule!

@anav: thank you so much for your assistance. And @TheCat12 earlier on.

CAPs is all working as expected. Only lurking issue I have now is just to implement a queue on the guest wifi to limit per-user bandwidth. I've just tried that now as a final part before starting to actually looking to push this out the door, but adding the queue with the 10.20.0.0 network as the target just seems to slam the brakes completely on internet for all users and make requests ridiculously slow across all 4 VLANs.

That said there's still something not quite right with internet access - for instance, even with no queues added, you can visit speedtest.net but it will take 30 seconds to think about starting the speed test. Google's speed test fails completely.

llamajaja · Tue Apr 16, 2024 4:30 pm

If you elect to do queuing, then disable fastrack. That accept thing got by all of us, good catch!!!

NetworqAndy · Tue Apr 16, 2024 4:37 pm

If you elect to do queuing, then disable fastrack. That accept thing got by all of us, good catch!!!

I've been caught by that before! Sadly even with the fasttrack rule disabled, even the basic set of (currently disabled) rules below makes the internet completely unworkable for any of the four vlans.

/queue type
add kind=pcq name=pcq-download-guest pcq-classifier=dst-address pcq-rate=10M
add kind=pcq name=pcq-upload-guest pcq-classifier=src-address pcq-rate=5M
/queue simple
add disabled=yes max-limit=900M/900M name=Global queue=\
    pcq-upload-default/pcq-download-default target=\
    10.10.0.0/16,10.20.0.0/16,10.30.0.0/16,10.40.0.0/16
add disabled=yes limit-at=700M/500M max-limit=700M/500M name=Guest parent=\
    Global queue=pcq-upload-guest/pcq-download-guest target=10.20.0.0/32

WAN failover - routes flapping [SOLVED]

WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping [SOLVED]

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Re: WAN failover - routes flapping

Who is online