From 0a09a6e6f247729457d15480f8d2b9bb0b89ae5e Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 29 Aug 2016 22:55:41 -0700 Subject: Updated (very out of date) docs and README.md --- docs/en/troubleshooting/where-to-look.html | 451 +++++++++++++++++++++++++++++ 1 file changed, 451 insertions(+) create mode 100644 docs/en/troubleshooting/where-to-look.html (limited to 'docs/en/troubleshooting/where-to-look.html') diff --git a/docs/en/troubleshooting/where-to-look.html b/docs/en/troubleshooting/where-to-look.html new file mode 100644 index 00000000..a1207aca --- /dev/null +++ b/docs/en/troubleshooting/where-to-look.html @@ -0,0 +1,451 @@ + + + + +Where to look - LEAP Platform Documentation + + + + + + + + +
+
+

Where to look for errors

+ +
The LEAP Platform is set of complementary packages and server recipes to automate the maintenance of LEAP services in a hardened Debian environment.
+
+
+ + +

General

+ +
    +
  • Please increase verbosity when debugging / filing issues in our issue tracker. You can do this with adding i.e. -v 5 after the leap cmd, i.e. leap -v 2 deploy.
    • +
  • We use the example.org domain for documentation purposes here, please replace it with the you domain.
    • +
+ + +

Firewall

+ +

Every node in your provider has its own restrictive firewall, but you might have a network firewall in place as well that is not managed by LEAP platform. To see what ports and addresses must be open, run this command:

+ +
workstation$ leap compile firewall
+
+ +

If any of those are blocked, then your provider will not work.

+ +

Webapp

+ +

Places to look for errors

+ +
    +
  • /var/log/apache2/error.log
    • +
  • /srv/leap/webapp/log/production.log
    • +
  • /var/log/syslog (watch out for stunnel issues)
    • +
  • /var/log/leap/*
    • +
+ + +

Is haproxy ok ?

+ +
curl -s -X  GET "http://127.0.0.1:4096"
+
+ +

Is couchdb accessible through stunnel ?

+ +
    +

  • Depending on how many couch nodes you have, increase the port for every test +(see /etc/haproxy/haproxy.cfg for the server/port mapping):

    + +

    curl -s -X GET “http://127.0.0.1:4000” + curl -s -X GET “http://127.0.0.1:4001” + …

    • +
+ + +

Check couchdb acl as admin

+ +
mkdir /etc/couchdb
+cat /srv/leap/webapp/config/couchdb.yml.admin  # see username and password
+echo "machine 127.0.0.1 login admin password <PASSWORD>" > /etc/couchdb/couchdb-admin.netrc
+chmod 600 /etc/couchdb/couchdb-admin.netrc
+
+curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096"
+curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096/_all_dbs"
+
+ +

Check couchdb acl as unpriviledged user

+ +
cat /srv/leap/webapp/config/couchdb.yml  # see username and password
+echo "machine 127.0.0.1 login webapp password <PASSWORD>" > /etc/couchdb/couchdb-webapp.netrc
+chmod 600 /etc/couchdb/couchdb-webapp.netrc
+
+curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4096"
+curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4096/_all_dbs"
+
+ +

All URLs accessible ?

+ + + + +

Check client config files

+ + + + +

Soledad

+ +
/var/log/soledad.log
+
+ +

Couchdb

+ +

Places to look for errors

+ +
    +
  • /var/log/couchdb/couch.log
    • +
  • /var/log/syslog (watch out for stunnel issues)
    • +
+ + +

Databases

+ +
    +
  • Following output shows all neccessary DBs that should be present. Note that the user-0123456.... DBs are the data stores for a particular user.
    • +
+ + +
+    curl -s --netrc-file /etc/couchdb/couchdb.netrc -X GET 'http://127.0.0.1:5984/_all_dbs'
+    ["customers","identities","sessions","shared","tickets","tokens","user-0","user-9d34680b01074c75c2ec58c7321f540c","user-9d34680b01074c75c2ec58c7325fb7ff","users"]
+
+ + +

Design Documents

+ +
    +
  • Is User _design doc available ?
    • +
+ + +
+    curl -s --netrc-file /etc/couchdb/couchdb.netrc -X  GET "http://127.0.0.1:5984/users/_design/User"
+
+ + +

Is couchdb cluster backend accessible through stunnel ?

+ +
    +
  • Find out how many connections are set up for the couchdb cluster backend:
    • +
+ + +
+    grep "accept = 127.0.0.1" /etc/stunnel/*
+
+ + +
    +
  • Now connect to all of those local endpoints to see if they up. All these tests should return “localhost [127.0.0.1] 4000 (?) open”
    • +
+ + +
+    nc -v 127.0.0.1 4000
+    nc -v 127.0.0.1 4001
+    ...
+
+ + +

MX

+ +

Places to look for errors

+ +
    +
  • /var/log/mail.log
    • +
  • /var/log/leap_mx.log
    • +
  • /var/log/syslog (watch out for stunnel issues)
    • +
+ + +

Is couchdb accessible through stunnel ?

+ +
    +

  • Depending on how many couch nodes you have, increase the port for every test +(see /etc/haproxy/haproxy.cfg for the server/port mapping):

    + +

    curl -s -X GET “http://127.0.0.1:4000” + curl -s -X GET “http://127.0.0.1:4001” + …

    • +
+ + +

Query leap-mx

+ +
    +
  • for useraccount
    • +
+ + +
+    postmap -v -q  "joe@dev.bitmask.net" tcp:localhost:2244
+    ...
+    postmap: dict_tcp_lookup: send: get jow@dev.bitmask.net
+    postmap: dict_tcp_lookup: recv: 200
+    ...
+
+ + +
    +
  • for mailalias
    • +
+ + +
+    postmap -v -q  "joe@dev.bitmask.net" tcp:localhost:4242
+    ...
+    postmap: dict_tcp_lookup: send: get joe@dev.bitmask.net
+    postmap: dict_tcp_lookup: recv: 200 f01bc1c70de7d7d80bc1ad77d987e73a
+    postmap: dict_tcp_lookup: found: f01bc1c70de7d7d80bc1ad77d987e73a
+    f01bc1c70de7d7d80bc1ad77d987e73a
+    ...
+
+ + +

Check couchdb acl as unpriviledged user

+ +
cat /etc/leap/mx.conf  # see username and password
+echo "machine 127.0.0.1 login leap_mx password <PASSWORD>" > /etc/couchdb/couchdb-leap_mx.netrc
+chmod 600 /etc/couchdb/couchdb-leap_mx.netrc
+
+curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4096/_all_dbs"   # pick one "user-<hash>" db
+curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4096/user-de9c77a3d7efbc779c6c20da88e8fb9c"
+
+ +
    +
  • you may check multiple times, cause 127.0.0.1:4096 is haproxy load-balancing the different couchdb nodes
    • +
+ + +

Mailspool

+ +
    +
  • Any file in the leap_mx mailspool longer for a few seconds ?
    • +
+ + +
+    ls -la /var/mail/vmail/Maildir/cur/
+
+ + +
    +
  • Any mails in postfix mailspool longer than a few seconds ?
    • +
+ + +
+    mailq
+
+ + +

Testing mail delivery

+ +
swaks -f alice@example.org -t bob@example.net -s mx1.example.net --port 25
+swaks -f varac@cdev.bitmask.net -t varac@cdev.bitmask.net -s chipmonk.cdev.bitmask.net --port 465 --tlsc
+swaks -f alice@example.org -t bob@example.net -s mx1.example.net --port 587 --tls
+
+ +

VPN

+ +

Places to look for errors

+ +
    +
  • /var/log/syslog (watch out for openvpn issues)
    • +
+ + +
+
+ + -- cgit v1.2.3