nitrolife

My provider doesn’t provide IPv6, but I rented a server in a data center, bought a subnet, and tunneled it home via WireGuard. So the scheme is roughly: VPS (fd00:1::/64) <-> (fd00:1::/64) Home router (realv6/64) <-> Home network

Router configuration:

/etc/sysctl.d/10-ipv6-privacy.conf

net.ipv6.conf.all.use_tempaddr = 0
net.ipv6.conf.default.use_tempaddr = 0
net.ipv6.conf.all.forwarding = 1
net.ipv6.conf.default.forwarding = 1

/etc/radvd.conf

interface br0 {
    AdvSendAdvert on;
    MinRtrAdvInterval 3;
    MaxRtrAdvInterval 30;

    AdvManagedFlag on;      # M=1 → Address via DHCPv6
    AdvOtherConfigFlag on;  # O=1 → Additional options via DHCPv6

    # SLAAC is still possible for Android
    prefix realv6::/64 {
        AdvOnLink on;
        AdvAutonomous on;   # Allow SLAAC
    };

    RDNSS realv6::1 {
        AdvRDNSSLifetime 1800;
    };
    DNSSL home.lan {
        AdvDNSSLLifetime 1800;
    };
};

/etc/kea/kea-dhcp6.conf

{
  "Dhcp6": {
    "interfaces-config": {
      "interfaces": [ "br0" ]
    },

    "lease-database": {
      "type": "memfile",
      "persist": true,
      "lfc-interval": 86400,
      "name": "/var/lib/kea/dhcp6.leases"
    },

    "renew-timer": 21600,
    "rebind-timer": 43200,
    "preferred-lifetime": 43200,
    "valid-lifetime": 86400,

    "subnet6": [
      {
        "id": 1,
        "subnet": "realv6::/64",
        "interface": "br0",
        "pools": [
          { "pool": "realv6::1000 - realv6::ffff" }
        ],
        "option-data": [
          { "name": "dns-servers",   "data": "realv6::1" },
          { "name": "domain-search", "data": "home.lan" }
        ]
      }
    ],

    "loggers": [
      {
        "name": "kea-dhcp6",
        "output-options": [
          { "output": "stdout" }
        ],
        "severity": "WARN"
      }
    ]
  }
}

And of course, iptables is necessary. Something like: /etc/iptables/ip6tables.rules

# Generated by ip6tables-save v1.6.0 on Thu Sep  8 13:29:11 2016
*nat
:PREROUTING ACCEPT [0:0]
:INPUT ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
COMMIT

*filter
:INPUT DROP [0:0]
:FORWARD DROP [0:0]
:OUTPUT ACCEPT [0:0]
#BASE INPUT
-A INPUT -i eno1 -j DROP
-A OUTPUT -o eno1 -j DROP
-A INPUT -i lo -j ACCEPT
-A INPUT -i br0 -j ACCEPT
-A INPUT -p ipv6-icmp -j ACCEPT
-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -i eno1 -j DROP
-A FORWARD -i br0 -j ACCEPT
-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -p ipv6-icmp -j ACCEPT
COMMIT

Yes, it is. But I have llama-swap, openweb-ui. If you spend some time on the llama-swap configuration, then the you have a good chance to run the model on 2 cards is through llama.cpp. The winnings, however, will not be x2 of course and will fall non-linearly from the number of cards. And you need motherboard with good PCI-E lines (2 pci-e x16 or more). But it’s still cheaper than one large card. Example:

HIP_VISIBLE_DEVICES=0,1 \
/opt/llama.cpp/build/bin/llama-server \
  --host 127.0.0.1 \
  --port 8082 \
  --model /storage/models/model.gguf \
  --n-gpu-layers all \
  --split-mode layer \
  --tensor-split 1,1 \
  --ctx-size 32768 \
  --batch-size 512 \
  --ubatch-size 512 \
  --flash-attn on \
  --parallel 1

There is a less stable but more productive one: --split-mode row

P.S. By the way, one RX9070XT on my instance translates posts and comments. You can test it if you want. =)

not a very popular opinion, but if you want an inexpensive, really inexpensive variant, take the AMD MX9070XT. AMD is not the most popular AI cards, but they are not bad with ROCm and for the price of 5090 you can put 5 cards (80 GB vram)