Removing large files; adding hook to watch for them

precommit-hooks/find-bad-ipam

@@ -17,10 +17,11 @@ function findBadTerm() {
     # param term: the term to search for duplicates 
     file="$1"
     term="$2"
-    results="$(grep "$term:" "$file" | sed 's/\s\+'"$term"':\s*//' | sort)"
+    results="$(grep -i "$term:" "$file" | tr '[[:upper:]]' '[[:lower:]]' | sed 's/\s+'"$term"':\s*//' | sort | uniq -c | grep -vE '^\s+1\s+' )"
 
-    if ! diff <(echo "$results") <(echo "$results" | uniq); then
-        echo "Some ${term^^} entries are duplicated. Search for the above terms in your inventory and deduplicate."
+    if [ -n "$results" ]; then
+        echo "Some ${term} entries are duplicated. Search for the above terms in your inventory and deduplicate."
+        echo "$results"
         exit 2
     fi
 }

precommit-hooks/find-data-files

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+result="$(find roles/*/{files,templates} -type f -exec file {} \; | grep -Ev ASCII\ text\|empty\|Unicode\ text)"
+if [ -n "$result" ]; then
+    echo "These files need to be evaluated -- generally, don't commit data files to Git."
+    echo "$result"
+    exit 1
+fi
+exit 0

precommit-hooks/find-large-files

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Limit files in git to 1M.
-find . -type f -exec du -k {} \; | egrep -v '^[[:digit:]]?[[:digit:]]?[[:digit:]][[:space:]]|\s./.git/'
+find . -type f -exec du -k {} \; | egrep -v '^[[:digit:]]?[[:digit:]]?[[:digit:]][[:space:]]|venv|\s./.git/'
 
 if [ $? -ne 1 ]; then
     echo

precommit-hooks/find-passwords-in-files

@@ -1,11 +1,9 @@
 #!/bin/bash
 
 # Ignore Ansibilized templates.
-saferegex='\s+}}"?\s*$'
+saferegex='\{\{.+\}\}'
 # Ignore comments
-saferegex="$saferegex"'|^[a-z,A-Z,0-9,_,-,/,.]+:\s*;|^[a-z,A-Z,0-9,_,-,/,.]+:\s*#|^[a-z,A-Z,0-9,_,-,/,.]+:\s*//'
-# Ignore binary file matches.
-saferegex="$saferegex"'|binary\ file\ matches'
+saferegex="$saferegex"'|^[a-z,A-Z,0-9,_,-,/,.]+:\s*;|^[a-z,A-Z,0-9,_,-,/,.]+:\s*#|^[a-z,A-Z,0-9,_,-,/,.]+:\s*//|\s+[/]?[*][/]?\s+'
 # AniNIX Constructs
 saferegex="$saferegex"'|password.aninix.net|aur.list'
 # Web constructs
@@ -18,8 +16,10 @@ saferegex="$saferegex"'|haveibeenpwned'
 saferegex="$saferegex"'|\s+=\s*$|\s+yes$|\s+no$'
 # Ignore LDAP attributes
 saferegex="$saferegex"'|pwpolicies|pwdLastSuccess|pwdAttribute|pwdMaxAge|pwdExpireWarning|pwdInHistory|pwdCheckQuality|pwdMaxFailure|pwdLockout|pwdLockoutDuration|pwdGraceAuthNLimit|pwdFailureCountInterval|pwdMustChange|pwdMinLength|pwdAllowUserChange|pwdSafeModify|pwdChangedTime|pwdPolicy|last changed their password on|/root/.ldappass'
+# Ignore IRC Modules
+saferegex="$saferegex"'|m_password_hash.so|/quote ns identify|SELECT|password_attribute|SET PASS|SASET PASS'
 
-egrep -ir 'secret|password|pw|passphrase' roles/*/{files,templates} 2>&1 | egrep -v "$saferegex"
+grep -iEr 'secret|password|pw|passphrase|pass=' roles/*/{files,templates} 2>&1 | egrep -v "$saferegex"
 if [ $? -ne 1 ]; then
     echo
     echo If these are false positives, you need to add the signature to the whitelist in $0.