Catching up with current successes

roles/Sharingan-Data/files/Core

@@ -0,0 +1,9 @@
+check program anixix-HTTP with path "/usr/lib/monitoring-plugins/check_http -H aninix.net -e 301"
+    if status == 1 then exec "/usr/bin/bash /etc/monit.d/scripts/notify aninix.net HTTP is CRITICAL"
+
+check program anixix-HTTPS with path "/usr/lib/monitoring-plugins/check_http -S -H aninix.net -e 200"
+    if status == 1 then exec "/usr/bin/bash /etc/monit.d/scripts/notify aninix.net HTTPS is CRITICAL"
+
+check program foundation-HTTPS with path "/usr/lib/monitoring-plugins/check_http -S -H foundation.aninix.net -e 200"
+    if status == 1 then exec "/usr/bin/bash /etc/monit.d/scripts/notify aninix.net HTTPS is CRITICAL"
+

roles/Sharingan-Data/files/scripts/notify

@@ -0,0 +1,2 @@
+#!/bin/bash
+systemd-cat -t sharingan-eval "$@"

roles/Sharingan-Data/files/sharingan-data.service/Archlinux

@@ -0,0 +1,19 @@
+[Unit]
+Description=AniNIX/Sharingan | Data filer
+Documentation=man:syslog-ng(8)
+Conflicts=emergency.service emergency.target
+Wants=network.target network-online.target
+After=network.target network-online.target
+
+[Service]
+Type=notify
+EnvironmentFile=-/etc/default/syslog-ng@sharingan-data
+EnvironmentFile=-/etc/sysconfig/syslog-ng@sharingan-data
+ExecStart=/usr/bin/syslog-ng -F $OTHER_OPTIONS --cfgfile $CONFIG_FILE --control $CONTROL_FILE --persist-file $PERSIST_FILE --pidfile $PID_FILE
+ExecReload=/usr/bin/kill -HUP $MAINPID
+StandardOutput=journal
+StandardError=journal
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target

roles/Sharingan-Data/files/sharingan-data.service/Debian

@@ -0,0 +1,19 @@
+[Unit]
+Description=AniNIX/Sharingan | Data filer
+Documentation=man:syslog-ng(8)
+Conflicts=emergency.service emergency.target
+Wants=network.target network-online.target
+After=network.target network-online.target
+
+[Service]
+Type=notify
+EnvironmentFile=-/etc/default/syslog-ng@sharingan-data
+EnvironmentFile=-/etc/sysconfig/syslog-ng@sharingan-data
+ExecStart=/usr/sbin/syslog-ng -F $OTHER_OPTIONS --cfgfile $CONFIG_FILE --control $CONTROL_FILE --persist-file $PERSIST_FILE --pidfile $PID_FILE
+ExecReload=/usr/bin/kill -HUP $MAINPID
+StandardOutput=journal
+StandardError=journal
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target

roles/Sharingan-Data/files/sharingan-eval.service

@@ -0,0 +1,33 @@
+# This file is systemd template for monit service. To
+# register monit with systemd, place the monit.service file
+# to the /lib/systemd/system/ directory and then start it
+# using systemctl (see bellow).
+#
+# Enable monit to start on boot:
+#         systemctl enable monit.service
+#
+# Start monit immediately:
+#         systemctl start monit.service
+#
+# Stop monit:
+#         systemctl stop monit.service
+#
+# Status:
+#         systemctl status monit.service
+
+[Unit]
+Description=AniNIX/Sharingan | Evaluation service, powered by monit
+After=network-online.target
+Documentation=man:monit(1) https://mmonit.com/wiki/Monit/HowTo
+
+[Service]
+Type=simple
+KillMode=process
+ExecStart=/usr/bin/monit -I
+ExecStop=/usr/bin/monit quit
+ExecReload=/usr/bin/monit reload
+Restart=on-abnormal
+StandardOutput=null
+
+[Install]
+WantedBy=multi-user.target

roles/Sharingan-Data/files/syslog-ng@sharingan-data

@@ -0,0 +1,5 @@
+CONFIG_FILE=/etc/syslog-ng/syslog-ng.conf
+PERSIST_FILE=/var/lib/syslog-ng/syslog-ng.persist
+CONTROL_FILE=/run/syslog-ng.ctl
+PID_FILE=/run/syslog-ng.pid
+OTHER_OPTIONS="--enable-core"

roles/Sharingan-Data/tasks/main.yml

@@ -5,6 +5,8 @@
      state: present
      name: 
        - syslog-ng
+       - monit
+       - monitoring-plugins
 
  - name: Sharingan-Data apps dir
    become: yes
@@ -14,6 +16,7 @@
 
  - name: Sharingan-Data include apps dir
    become: yes
+   register: base_config
    lineinfile: 
      path: /etc/syslog-ng/syslog-ng.conf
      line: "{{ item }}"
@@ -29,31 +32,75 @@
      owner: root
      group: root
      mode: 0750
+ - name: Sharingan-Data service conf
+   become: yes
+   copy:
+     src: syslog-ng@sharingan-data
+     dest: /etc/default/syslog-ng@sharingan-data
+     owner: root
+     group: root
+     mode: 0655
 
  - name: Sharingan-Data filer service
    become: yes
    copy:
-     remote_src: yes
-     src: /usr/lib/systemd/system/syslog-ng@.service
+     src: "sharingan-data.service/{{ ansible_os_family }}"
      dest: /usr/lib/systemd/system/sharingan-data.service
      owner: root
      group: root
      mode: 0750
 
- - name: Sharingan-Data replace content
+ - name: Sharingan-Eval service
    become: yes
-   replace:
-     path: /usr/lib/systemd/system/sharingan-data.service
-     regexp: '%i'
-     replace: 'default'
+   copy:
+     src: sharingan-eval.service
+     dest: /usr/lib/systemd/system/sharingan-eval.service
+     owner: root
+     group: root
+     mode: 0750
 
- - name: Sharingan-Data set vanity description
+ - name: Sharingan-Eval monitrc
    become: yes
-   lineinfile:
-     path: /usr/lib/systemd/system/sharingan-data.service
-     regexp: 'Description='
-     line: 'Description=AniNIX/Sharingan | Data filer'
+   template:
+     src: monitrc.j2
+     dest: /etc/monitrc
+     owner: root
+     group: root
+     mode: 0700
 
+ - name: Sharingan-Eval includes dir
+   become: yes
+   file:
+     path: /etc/monit.d
+     state: directory
+
+ - name: Sharingan-Eval monit templates
+   become: yes
+   copy:
+     src: templates
+     dest: /etc/monit.d/templates
+     owner: root
+     group: root
+     mode: 0700
+
+ - name: Sharingan-Eval monit scripts
+   become: yes
+   copy:
+     src: templates
+     dest: /etc/monit.d/scripts
+     owner: root
+     group: root
+     mode: 0700
+
+
+ - name: Sharingan-Eval monit host config
+   become: yes
+   copy:
+     src: "{{ inventory_hostname }}"
+     dest: "/etc/monit.d/{{ inventory_hostname }}"
+     owner: root
+     group: root
+     mode: 0700
 
  - name: Sharingan-Data heartbeat service
    become: yes
@@ -81,14 +128,16 @@
    become: yes
    service:
      name: "{{ item }}"
-     state: started
+     state: restarted
      enabled: yes
    loop:
-     - sharingan-heartbeat.timer
      - sharingan-data.service
+     - sharingan-heartbeat.timer
+     - sharingan-eval.service
 
  - name: Disable default service
    become: yes
+   ignore_errors: yes
    service:
      name: syslog-ng@default.service
      state: stopped

roles/Sharingan-Data/templates/monitrc.j2

@@ -0,0 +1,300 @@
+###############################################################################
+## Monit control file
+###############################################################################
+##
+## Comments begin with a '#' and extend through the end of the line. Keywords
+## are case insensitive. All path's MUST BE FULLY QUALIFIED, starting with '/'.
+##
+## Below you will find examples of some frequently used statements. For
+## information about the control file and a complete list of statements and
+## options, please have a look in the Monit manual.
+##
+##
+###############################################################################
+## Global section
+###############################################################################
+set daemon  30              # check services at 30 seconds intervals
+    with start delay 240    # optional: delay the first check by 4-minutes (by
+                            # default Monit check immediately after Monit start)
+#
+## Set syslog logging. If you want to log to a standalone log file instead,
+## specify the full path to the log file
+#
+set log syslog
+
+#
+#
+## Set the location of the Monit lock file which stores the process id of the
+## running Monit instance. By default this file is stored in $HOME/.monit.pid
+#
+# set pidfile /var/run/monit.pid
+#
+## Set the location of the Monit id file which stores the unique id for the
+## Monit instance. The id is generated and stored on first Monit start. By
+## default the file is placed in $HOME/.monit.id.
+#
+# set idfile /var/.monit.id
+#
+## Set the location of the Monit state file which saves monitoring states
+## on each cycle. By default the file is placed in $HOME/.monit.state. If
+## the state file is stored on a persistent filesystem, Monit will recover
+## the monitoring state across reboots. If it is on temporary filesystem, the
+## state will be lost on reboot which may be convenient in some situations.
+#
+# set statefile /var/.monit.state
+#
+#
+
+## Set limits for various tests. The following example shows the default values:
+##
+# set limits {
+#     programOutput:     512 B,      # check program's output truncate limit
+#     sendExpectBuffer:  256 B,      # limit for send/expect protocol test
+#     fileContentBuffer: 512 B,      # limit for file content test
+#     httpContentBuffer: 1 MB,       # limit for HTTP content test
+#     networkTimeout:    5 seconds   # timeout for network I/O
+#     programTimeout:    300 seconds # timeout for check program
+#     stopTimeout:       30 seconds  # timeout for service stop
+#     startTimeout:      30 seconds  # timeout for service start
+#     restartTimeout:    30 seconds  # timeout for service restart
+# }
+
+## Set global SSL options (just most common options showed, see manual for
+## full list).
+#
+# set ssl {
+#     verify     : enable, # verify SSL certificates (disabled by default but STRONGLY RECOMMENDED)
+#     selfsigned : allow   # allow self signed SSL certificates (reject by default)
+#}
+#
+#
+## Set the list of mail servers for alert delivery. Multiple servers may be
+## specified using a comma separator. If the first mail server fails, Monit
+# will use the second mail server in the list and so on. By default Monit uses
+# port 25 - it is possible to override this with the PORT option.
+#
+# set mailserver mail.bar.baz,               # primary mailserver
+#                backup.bar.baz port 10025,  # backup mailserver on port 10025
+#                localhost                   # fallback relay
+#
+#
+## By default Monit will drop alert events if no mail servers are available.
+## If you want to keep the alerts for later delivery retry, you can use the
+## EVENTQUEUE statement. The base directory where undelivered alerts will be
+## stored is specified by the BASEDIR option. You can limit the queue size
+## by using the SLOTS option (if omitted, the queue is limited by space
+## available in the back end filesystem).
+#
+# set eventqueue
+#     basedir /var/monit  # set the base directory where events will be stored
+#     slots 100           # optionally limit the queue size
+#
+#
+## Send status and events to M/Monit (for more information about M/Monit
+## see https://mmonit.com/). By default Monit registers credentials with
+## M/Monit so M/Monit can smoothly communicate back to Monit and you don't
+## have to register Monit credentials manually in M/Monit. It is possible to
+## disable credential registration using the commented out option below.
+## Though, if safety is a concern we recommend instead using https when
+## communicating with M/Monit and send credentials encrypted. The password
+## should be URL encoded if it contains URL-significant characters like
+## ":", "?", "@". Default timeout is 5 seconds, you can customize it by
+## adding the timeout option.
+#
+# set mmonit http://monit:monit@192.168.1.10:8080/collector
+#     # with timeout 30 seconds              # Default timeout is 5 seconds
+#     # and register without credentials     # Don't register credentials
+#
+#
+## Monit by default uses the following format for alerts if the mail-format
+## statement is missing::
+## --8<--
+## set mail-format {
+##   from:    Monit <monit@$HOST>
+##   subject: monit alert --  $EVENT $SERVICE
+##   message: $EVENT Service $SERVICE
+##                 Date:        $DATE
+##                 Action:      $ACTION
+##                 Host:        $HOST
+##                 Description: $DESCRIPTION
+##
+##            Your faithful employee,
+##            Monit
+## }
+## --8<--
+##
+## You can override this message format or parts of it, such as subject
+## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc.
+## are expanded at runtime. For example, to override the sender, use:
+#
+# set mail-format { from: monit@foo.bar }
+#
+#
+## You can set alert recipients whom will receive alerts if/when a
+## service defined in this file has errors. Alerts may be restricted on
+## events by using a filter as in the second example below.
+#
+# set alert sysadm@foo.bar                       # receive all alerts
+#
+## Do not alert when Monit starts, stops or performs a user initiated action.
+## This filter is recommended to avoid getting alerts for trivial cases.
+#
+# set alert your-name@your.domain not on { instance, action }
+#
+#
+## Monit has an embedded HTTP interface which can be used to view status of
+## services monitored and manage services from a web interface. The HTTP
+## interface is also required if you want to issue Monit commands from the
+## command line, such as 'monit status' or 'monit restart service' The reason
+## for this is that the Monit client uses the HTTP interface to send these
+## commands to a running Monit daemon. See the Monit Wiki if you want to
+## enable SSL for the HTTP interface.
+#
+set httpd port 2812 and
+    use address localhost  # only accept connection from localhost (drop if you use M/Monit)
+    allow localhost        # allow localhost to connect to the server and
+    allow admin:"{{ monitcli | default('monit') }}"      # require user 'admin' with password 'monit'
+    #with ssl {            # enable SSL/TLS and set path to server certificate
+    #    pemfile: /etc/ssl/certs/monit.pem
+    #}
+
+###############################################################################
+## Services
+###############################################################################
+##
+## Check general system resources such as load average, cpu and memory
+## usage. Each test specifies a resource, conditions and the action to be
+## performed should a test fail.
+#
+#  check system $HOST
+#    if loadavg (1min) per core > 2 for 5 cycles then alert
+#    if loadavg (5min) per core > 1.5 for 10 cycles then alert
+#    if cpu usage > 95% for 10 cycles then alert
+#    if memory usage > 75% then alert
+#    if swap usage > 25% then alert
+#
+#
+## Check if a file exists, checksum, permissions, uid and gid. In addition
+## to alert recipients in the global section, customized alert can be sent to
+## additional recipients by specifying a local alert handler. The service may
+## be grouped using the GROUP option. More than one group can be specified by
+## repeating the 'group name' statement.
+#
+#  check file apache_bin with path /usr/local/apache/bin/httpd
+#    if failed checksum and
+#       expect the sum 8f7f419955cefa0b33a2ba316cba3659 then unmonitor
+#    if failed permission 755 then unmonitor
+#    if failed uid "root" then unmonitor
+#    if failed gid "root" then unmonitor
+#    alert security@foo.bar on {
+#           checksum, permission, uid, gid, unmonitor
+#        } with the mail-format { subject: Alarm! }
+#    group server
+#
+#
+## Check that a process is running, in this case Apache, and that it respond
+## to HTTP and HTTPS requests. Check its resource usage such as cpu and memory,
+## and number of children. If the process is not running, Monit will restart
+## it by default. In case the service is restarted very often and the
+## problem remains, it is possible to disable monitoring using the TIMEOUT
+## statement. This service depends on another service (apache_bin) which
+## is defined above.
+#
+#  check process apache with pidfile /usr/local/apache/logs/httpd.pid
+#    start program = "/etc/init.d/httpd start" with timeout 60 seconds
+#    stop program  = "/etc/init.d/httpd stop"
+#    if cpu > 60% for 2 cycles then alert
+#    if cpu > 80% for 5 cycles then restart
+#    if totalmem > 200.0 MB for 5 cycles then restart
+#    if children > 250 then restart
+#    if disk read > 500 kb/s for 10 cycles then alert
+#    if disk write > 500 kb/s for 10 cycles then alert
+#    if failed host www.tildeslash.com port 80 protocol http and request "/somefile.html" then restart
+#    if failed port 443 protocol https with timeout 15 seconds then restart
+#    if 3 restarts within 5 cycles then unmonitor
+#    depends on apache_bin
+#    group server
+#
+#
+## Check filesystem permissions, uid, gid, space usage, inode usage and disk I/O.
+## Other services, such as databases, may depend on this resource and an automatically
+## graceful stop may be cascaded to them before the filesystem will become full and data
+## lost.
+#
+#  check filesystem datafs with path /dev/sdb1
+#    start program  = "/bin/mount /data"
+#    stop program  = "/bin/umount /data"
+#    if failed permission 660 then unmonitor
+#    if failed uid "root" then unmonitor
+#    if failed gid "disk" then unmonitor
+#    if space usage > 80% for 5 times within 15 cycles then alert
+#    if space usage > 99% then stop
+#    if inode usage > 30000 then alert
+#    if inode usage > 99% then stop
+#    if read rate > 1 MB/s for 5 cycles then alert
+#    if read rate > 500 operations/s for 5 cycles then alert
+#    if write rate > 1 MB/s for 5 cycles then alert
+#    if write rate > 500 operations/s for 5 cycles then alert
+#    if service time > 10 milliseconds for 3 times within 5 cycles then alert
+#    group server
+#
+#
+## Check a file's timestamp. In this example, we test if a file is older
+## than 15 minutes and assume something is wrong if its not updated. Also,
+## if the file size exceed a given limit, execute a script
+#
+#  check file database with path /data/mydatabase.db
+#    if failed permission 700 then alert
+#    if failed uid "data" then alert
+#    if failed gid "data" then alert
+#    if timestamp > 15 minutes then alert
+#    if size > 100 MB then exec "/my/cleanup/script" as uid dba and gid dba
+#
+#
+## Check directory permission, uid and gid.  An event is triggered if the
+## directory does not belong to the user with uid 0 and gid 0.  In addition,
+## the permissions have to match the octal description of 755 (see chmod(1)).
+#
+#  check directory bin with path /bin
+#    if failed permission 755 then unmonitor
+#    if failed uid 0 then unmonitor
+#    if failed gid 0 then unmonitor
+#
+#
+## Check a remote host availability by issuing a ping test and check the
+## content of a response from a web server. Up to three pings are sent and
+## connection to a port and an application level network check is performed.
+#
+#  check host myserver with address 192.168.1.1
+#    if failed ping then alert
+#    if failed port 3306 protocol mysql with timeout 15 seconds then alert
+#    if failed port 80 protocol http
+#       and request /some/path with content = "a string"
+#    then alert
+#
+#
+## Check a network link status (up/down), link capacity changes, saturation
+## and bandwidth usage.
+#
+#  check network public with interface eth0
+#    if failed link then alert
+#    if changed link then alert
+#    if saturation > 90% then alert
+#    if download > 10 MB/s then alert
+#    if total uploaded > 1 GB in last hour then alert
+#
+#
+## Check custom program status output.
+#
+#  check program myscript with path /usr/local/bin/myscript.sh
+#    if status != 0 then alert
+#
+#
+###############################################################################
+## Includes
+###############################################################################
+##
+## It is possible to include additional configuration parts from other files or
+## directories.
+#
+include /etc/monit.d/{{ inventory_hostname }}